Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's Git administration access, network and repository coordination \
526         before retrying; an unattended seat cannot approve those permissions"
527    )
528}
529
530/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
531/// on an early return or a panic.
532///
533/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
534/// manual `magi fix` invocations against the same run are otherwise
535/// invisible to each other and would race to remove and recreate the same
536/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
537/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
538/// which is only ever reclaimed later, out of band, by
539/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
540/// `magi fix` invocation is not necessarily running under either of those, so
541/// nothing would ever sweep a lock a killed or crashed process left behind.
542/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
543/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
544/// lease use: an unreadable or unparsable pid, or a liveness query the
545/// platform cannot answer, reads as alive and the lock is left in place.
546struct FixClaim {
547    path: PathBuf,
548}
549
550impl FixClaim {
551    fn acquire(dir: &Path) -> Result<Self> {
552        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553        let path = dir.join("fix.lock");
554        let retry = crate::lock_file::Retry::new(None);
555        match Self::create(&path, retry) {
556            Ok(claim) => Ok(claim),
557            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558                if Self::reclaim_if_dead(&path) {
559                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560                } else {
561                    bail!(
562                        "another repair or `magi fix` is already running for this run ({} exists)",
563                        path.display()
564                    )
565                }
566            }
567            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568        }
569    }
570
571    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572        let mut f = retry.create(path)?;
573        use std::io::Write as _;
574        // Read back by `reclaim_if_dead` on a later, stuck invocation.
575        writeln!(f, "{}", std::process::id())?;
576        Ok(Self {
577            path: path.to_owned(),
578        })
579    }
580
581    /// True if the lock named a process confirmed dead, in which case it was
582    /// also removed. Never true on an unreadable file, an unparsable pid, or
583    /// a liveness query the platform cannot answer — see this type's own doc.
584    fn reclaim_if_dead(path: &Path) -> bool {
585        let dead = std::fs::read_to_string(path)
586            .ok()
587            .and_then(|body| body.trim().parse::<u32>().ok())
588            .is_some_and(|pid| !crate::proc::pid_alive(pid));
589        dead && std::fs::remove_file(path).is_ok()
590    }
591}
592
593impl Drop for FixClaim {
594    fn drop(&mut self) {
595        let _ = std::fs::remove_file(&self.path);
596    }
597}
598
599/// Holds the mutable runner as well as its durable lease: cancellation must
600/// clear both copies of ownership, including state a caller may save later.
601struct DriverExecution<'a> {
602    runner: &'a mut Runner,
603    driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607    fn drop(&mut self) {
608        self.driver.release_local(&mut self.runner.state);
609    }
610}
611
612impl Runner {
613    /// Start a fresh run against `repo`.
614    pub async fn start(
615        repo: &Path,
616        instruction: String,
617        config: Config,
618        origin: Origin,
619    ) -> Result<Self> {
620        Self::start_naming(repo, instruction, "", config, origin).await
621    }
622
623    /// [`Runner::start`] for a queued task. Titles and instructions never
624    /// select implementation sources. The title argument is retained for callers.
625    pub async fn start_naming(
626        repo: &Path,
627        instruction: String,
628        _also_scan: &str,
629        config: Config,
630        origin: Origin,
631    ) -> Result<Self> {
632        let repo = git::toplevel(repo).await?;
633        let missing = agent::missing_programs(&config.agents);
634        if !missing.is_empty() {
635            bail!(
636                "these agent programs are not on PATH: {}. Fix the roster in \
637                 magi.toml or install them.",
638                missing.join(", ")
639            );
640        }
641        let base_branch =
642            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643                .await?;
644        let fetched_base = resolve_base(
645            &repo,
646            &base_branch,
647            &config.merge.remote,
648            config.coordination.as_ref(),
649        )
650        .await?;
651        let base_commit = fetched_base.commit.clone();
652        let roles = config.resolve_roles()?;
653        let max_parallel = config.graph.max_parallel.max(1);
654        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655        // Recorded before the first save, so a crash right after minting
656        // cannot leave a run with no origin. Legibility only: nothing reads it
657        // to decide anything.
658        state.fetched_base = Some(fetched_base);
659        state.origin = Some(origin);
660        state.event("start", format!("run {} created", state.id));
661        state.save()?;
662        Ok(Self {
663            state,
664            roles,
665            sem: Arc::new(Semaphore::new(max_parallel)),
666            pause: Pause::new(),
667            interrupt: Pause::new(),
668        })
669    }
670
671    /// Open a review-only run against work that already exists on `branch`.
672    ///
673    /// The expensive half of the graph is the implement wave — measured at
674    /// 111 and 134 internal tool-loop turns on this repository, against a
675    /// handful for a judge or a reviewer. The cheap half is worth running on
676    /// hand-written work too, and there was no way to reach it.
677    ///
678    /// No new state and no schema change are needed: a run with **one** viable
679    /// candidate and a tally already decided degrades `execute` to exactly
680    /// review → gate → merge, because `judge` skips a single-candidate field,
681    /// `deliberate` has fewer than two first choices to reconcile, `vote`
682    /// returns early, `tally` is already present and `fold_losers` has no
683    /// losers. Resuming such a run therefore does the right thing as well.
684    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685        Self::review_taking_over(repo, branch, config, None, origin).await
686    }
687
688    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
689    /// at the same task still has `branch` checked out, its worktree is
690    /// released first if that is safe (see [`crate::handover`]), and the
691    /// review refuses with the reason if it is not. `None` is a hand-run
692    /// review: it has no earlier attempts, so only a worktree of a dead run
693    /// magi recorded itself can be released.
694    pub async fn review_taking_over(
695        repo: &Path,
696        branch: &str,
697        config: Config,
698        takeover: Option<crate::handover::Takeover>,
699        origin: Origin,
700    ) -> Result<Self> {
701        let repo = git::toplevel(repo).await?;
702        let missing = agent::missing_programs(&config.agents);
703        if !missing.is_empty() {
704            bail!(
705                "these agent programs are not on PATH: {}. Fix the roster in \
706                 magi.toml or install them.",
707                missing.join(", ")
708            );
709        }
710        let base_branch =
711            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712                .await?;
713        if base_branch == branch {
714            bail!("`{branch}` is the base branch; there is nothing to review against");
715        }
716        let fetched_base = resolve_base(
717            &repo,
718            &base_branch,
719            &config.merge.remote,
720            config.coordination.as_ref(),
721        )
722        .await?;
723        let base_commit = fetched_base.commit.clone();
724
725        let roles = config.resolve_roles()?;
726        let max_parallel = config.graph.max_parallel.max(1);
727        let mut state = RunState::new(
728            repo.clone(),
729            base_branch,
730            base_commit.clone(),
731            String::new(),
732            config,
733        );
734        state.fetched_base = Some(fetched_base);
735        state.origin = Some(origin);
736
737        // Released before anything else touches the branch: a stale local
738        // branch is moved with `git branch -f`, which git refuses while an
739        // earlier attempt's worktree still has it checked out. Everything
740        // after this point that can fail puts the old run back.
741        // A hand-run review has no task, hence no earlier attempts, but a
742        // worktree of a dead run magi made may still be released.
743        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744            earlier: Vec::new(),
745            home: crate::run::home(),
746            choice: None,
747            task_instruction: None,
748        });
749        let wait = repository_wait(&state, None);
750        let coordination_config = state.config.coordination.clone();
751        let coordination =
752            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753                .await?;
754        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755        if let Some(released) = &released {
756            state.event(
757                "release",
758                format!(
759                    "took `{branch}` over from run {}: its worktree was released: {}",
760                    crate::run::short_of(&released.old_id),
761                    released.audit
762                ),
763            );
764        }
765        // The owner's answer to an earlier divergence question is applied
766        // here: after the release (git will not move a checked-out branch)
767        // and before the sync that would otherwise ask again.
768        if let Some(choice) = takeover.choice.as_ref()
769            && let Err(e) =
770                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771                    .await
772        {
773            if let Some(released) = &released {
774                released.restore(&repo, branch).await;
775            }
776            return Err(e.context("applying the owner's answer about the diverged branch"));
777        }
778        // An earlier attempt of this task that published the task head, and
779        // whose work is the branch under review, hands that head over: the
780        // review run updates the same pull request. Without one (a legacy
781        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
782        // own head and its own open pull request is adopted as before.
783        for id in takeover.earlier.iter().rev() {
784            if let Ok(old) = RunState::load_under(id, &takeover.home)
785                && let Some(head) = old.pr_head.clone()
786                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787            {
788                state.pr_head = Some(head);
789                break;
790            }
791        }
792        drop(coordination);
793        let opened = Self::open_review(
794            &repo,
795            branch,
796            state,
797            roles,
798            max_parallel,
799            base_commit,
800            &takeover,
801        )
802        .await;
803        if opened.is_err()
804            && let Some(released) = &released
805        {
806            let _coordination = crate::repo_lock::SeatLease::operation_with(
807                &repo,
808                coordination_config.as_ref(),
809                &wait,
810            )
811            .await?;
812            released.restore(&repo, branch).await;
813        }
814        opened
815    }
816
817    /// Recover exactly the operator-selected public implementation, without touching old sources.
818    pub async fn recover_pr(
819        repo: &Path,
820        task: &str,
821        selection: crate::pr_recovery::Selection,
822        mut config: Config,
823    ) -> Result<Self> {
824        selection.configure(&mut config)?;
825        selection.check_task(task, &selection.instruction)?;
826        let base_branch =
827            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828                .await?;
829        let fetched_base = resolve_base(
830            repo,
831            &base_branch,
832            &config.merge.remote,
833            config.coordination.as_ref(),
834        )
835        .await?;
836        let base_commit = fetched_base.commit.clone();
837        selection
838            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839            .await?;
840        Box::pin(selection.verify_source(repo)).await?;
841        if let Some(proof) = &selection.source_proof {
842            anyhow::ensure!(
843                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844                "source rebase base is outside the fetched landing base"
845            );
846        }
847        let roles = config.resolve_roles()?;
848        let max_parallel = config.graph.max_parallel.max(1);
849        let mut state = RunState::new(
850            repo.to_path_buf(),
851            base_branch,
852            base_commit.clone(),
853            selection.instruction.clone(),
854            config,
855        );
856        state.fetched_base = Some(fetched_base);
857        state.origin = Some(crate::run::Origin::queue(task));
858        state.pr_head = Some(selection.branch.clone());
859        state.pr_recovery = Some(selection.clone());
860        crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862    }
863
864    async fn open_selected(
865        repo: &Path,
866        state: RunState,
867        roles: ResolvedRoles,
868        max_parallel: usize,
869        base_commit: String,
870        selection: crate::pr_recovery::Selection,
871    ) -> Result<Self> {
872        let mut state = state;
873        let branch = state.branch_for('A');
874        let wait = repository_wait(&state, None);
875        let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876        git::git(repo, &["branch", &branch, selection.source()]).await?;
877        drop(coordination);
878        state.event(
879            "recovery",
880            format!(
881                "selected implementation {} for {} (public lease {})",
882                selection.source(),
883                selection.url,
884                selection.head
885            ),
886        );
887        let takeover = crate::handover::Takeover {
888            earlier: Vec::new(),
889            home: crate::run::home(),
890            choice: None,
891            task_instruction: Some(selection.instruction.clone()),
892        };
893        state.save()?;
894        let id = state.id.clone();
895        let opened = Self::open_review(
896            repo,
897            &branch,
898            state,
899            roles,
900            max_parallel,
901            base_commit,
902            &takeover,
903        )
904        .await;
905        match opened {
906            Ok(runner) => Ok(runner),
907            Err(e) => {
908                if let Ok(mut partial) = RunState::load(&id) {
909                    partial.status = RunStatus::Blocked;
910                    partial.driver_exited = true;
911                    partial.event(
912                        "recovery",
913                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
914                    );
915                    partial.save()?;
916                }
917                Err(crate::pr_recovery::StartupFailed {
918                    run: id,
919                    cause: format!("{e:#}"),
920                }
921                .into())
922            }
923        }
924    }
925
926    /// What the newest earlier attempt that reviewed `branch` leaves for this
927    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
928    /// no earlier attempts. Reads the earlier record only: ownership of its
929    /// worktree and branch is [`crate::handover`]'s business.
930    async fn inherit(
931        repo: &Path,
932        branch: &str,
933        base_commit: &str,
934        takeover: &crate::handover::Takeover,
935    ) -> Option<crate::recovery::RecoveryRecord> {
936        use crate::recovery::{Relation, assess};
937        let newest = takeover.earlier.last()?;
938        let mut chosen = None;
939        for id in takeover.earlier.iter().rev() {
940            if let Ok(old) = RunState::load_under(id, &takeover.home)
941                && !old.reviews.is_empty()
942                && (old.winner().is_some_and(|w| w.branch == branch)
943                    || old.pr_head.as_deref() == Some(branch))
944            {
945                chosen = Some(old);
946                break;
947            }
948        }
949        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950            .await
951            .unwrap_or_default();
952        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953            Some(last) if last.head == tip => Relation::Same,
954            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955            _ => Relation::Diverged,
956        };
957        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958        Some(assess(from, chosen.as_ref(), base_commit, relation))
959    }
960
961    /// The half of [`Runner::review_taking_over`] that can fail after an
962    /// earlier attempt's worktree was released.
963    async fn open_review(
964        repo: &Path,
965        branch: &str,
966        mut state: RunState,
967        roles: ResolvedRoles,
968        max_parallel: usize,
969        base_commit: String,
970        takeover: &crate::handover::Takeover,
971    ) -> Result<Self> {
972        let wait = repository_wait(&state, None);
973        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974        if state.pr_recovery.is_none() {
975            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976        }
977        // The commit subjects are the closest thing to a task statement that
978        // existing work carries, and the reviewers are told as much.
979        let start = review_base(
980            repo,
981            &state.config.merge.remote,
982            &state.base_branch,
983            &base_commit,
984            branch,
985        )
986        .await;
987        let log = git::log_oneline(repo, &start, branch)
988            .await
989            .unwrap_or_default();
990        let log_text = if log.trim().is_empty() {
991            "(no commit messages)"
992        } else {
993            log.trim()
994        };
995        let mut instruction = format!(
996            "Review the work already on branch `{branch}`. There is no task \
997             statement: what the change claims to do is whatever its commits \
998             say.\n\n{log_text}"
999        );
1000        // A retry of a task whose earlier attempt left reviews behind carries
1001        // that attempt's evidence, and the task's own statement with it: the
1002        // completion criteria are the task's, not whatever the commits claim.
1003        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004            if let Some(task) = takeover
1005                .task_instruction
1006                .as_deref()
1007                .filter(|t| !t.trim().is_empty())
1008                && rec.reviewed_head.is_some()
1009            {
1010                instruction = format!(
1011                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012                    task.trim()
1013                );
1014                rec.criteria_inherited = true;
1015            }
1016            state.event("recovery", rec.summary());
1017            state.recovery = Some(rec);
1018        }
1019        if let Some(selected) = &state.pr_recovery {
1020            instruction = selected.instruction.clone();
1021        }
1022        state.task_instruction = takeover.task_instruction.clone();
1023        state.instruction = instruction;
1024        state.reviewed_commits = Some(
1025            git::subjects(repo, &start, branch)
1026                .await
1027                .unwrap_or_default(),
1028        );
1029
1030        // An attached worktree, so the fixer's commits land on the branch under
1031        // review rather than on a detached head nobody will look at again.
1032        let worktree = state.worktree_root().join("under-review");
1033        if let Some(parent) = worktree.parent() {
1034            tokio::fs::create_dir_all(parent).await.ok();
1035        }
1036        let path = worktree.to_string_lossy().to_string();
1037        git::git(repo, &["worktree", "add", &path, branch])
1038            .await
1039            .with_context(|| {
1040                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1041            })?;
1042
1043        let setup = match ensure_setup_config(&mut state, repo).await {
1044            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1045            Err(e) => Err(e),
1046        };
1047        if let Err(e) = setup {
1048            git::worktree_remove_clean(repo, &worktree).await.ok();
1049            return Err(e);
1050        }
1051        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1052            .await
1053            .unwrap_or(0);
1054        if commits == 0 {
1055            git::worktree_remove_clean(repo, &worktree).await.ok();
1056            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1057        }
1058        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1059            .await
1060            .map(|f| f.len())
1061            .unwrap_or(0);
1062        if files == 0
1063            && let (Ok(head_tree), Ok(base_tree)) = (
1064                git::tree_of(&worktree, "HEAD").await,
1065                git::tree_of(&worktree, &base_commit).await,
1066            )
1067            && head_tree == base_tree
1068        {
1069            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1070            git::worktree_remove_clean(repo, &worktree).await.ok();
1071            bail!(
1072                "`{branch}` at {} has a tree identical to base {}; this usually means \
1073                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1074                 against `{}/{branch}`) rather than an empty change",
1075                short(&head),
1076                short(&base_commit),
1077                state.config.merge.remote
1078            );
1079        }
1080        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1081            .await
1082            .unwrap_or_default();
1083
1084        state.candidates.push(Candidate {
1085            index: 0,
1086            label: 'A',
1087            // Not an agent id on purpose: nothing in the roster wrote this, and
1088            // the stats tables must not credit anyone with a win for it.
1089            agent: EXISTING_BRANCH.to_owned(),
1090            branch: branch.to_owned(),
1091            worktree,
1092            summary: String::new(),
1093            stat,
1094            files,
1095            commits,
1096            empty: false,
1097            failed: None,
1098            verified_noop: None,
1099            duration_ms: 0,
1100            folded: false,
1101        });
1102        state.tally = Some(Tally {
1103            first_choice: BTreeMap::from([('A', 0)]),
1104            borda: BTreeMap::new(),
1105            winner: 'A',
1106            rankings: 0,
1107            unanimous_initial: false,
1108            deliberated: false,
1109            changed_votes: 0,
1110            unanimous_final: false,
1111            tie_break: None,
1112            // No panel sat, so no quorum applies. Zero judges is the correct
1113            // number for work that never competed, and must not be reported as
1114            // a collapsed panel.
1115            judges: 0,
1116            present: 0,
1117            quorum: 0,
1118            met_quorum: true,
1119            uncontested: Some("review-only run: nothing competed".to_owned()),
1120        });
1121        state.status = RunStatus::Reviewing;
1122        state.prep_complete = Some(true);
1123        state.event(
1124            "start",
1125            format!(
1126                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1127                state.id
1128            ),
1129        );
1130        state.save()?;
1131        Ok(Self {
1132            state,
1133            roles,
1134            sem: Arc::new(Semaphore::new(max_parallel)),
1135            pause: Pause::new(),
1136            interrupt: Pause::new(),
1137        })
1138    }
1139
1140    /// Reopen an existing run.
1141    pub fn resume(id: &str) -> Result<Self> {
1142        let state = RunState::load(id)?;
1143        crate::pr_head::ensure_owner(&state)?;
1144        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1145            bail!(
1146                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1147            );
1148        }
1149        if let Some(to) = &state.released_to {
1150            bail!(
1151                "run {} cannot be resumed: its worktree was released to run {}",
1152                state.short(),
1153                crate::run::short_of(to)
1154            );
1155        }
1156        let roles = state.config.resolve_roles()?;
1157        if let Some(id) = &state.config.graph.selected_implementer
1158            && state.candidates.iter().any(|c| c.agent != *id)
1159        {
1160            bail!(
1161                "replacement implementer pin does not match the saved candidates; task must stay held"
1162            );
1163        }
1164        let max_parallel = state.config.graph.max_parallel.max(1);
1165        Ok(Self {
1166            state,
1167            roles,
1168            sem: Arc::new(Semaphore::new(max_parallel)),
1169            pause: Pause::new(),
1170            interrupt: Pause::new(),
1171        })
1172    }
1173
1174    /// Walk the graph to a terminal state, skipping nodes already recorded.
1175    ///
1176    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1177    /// the phone - ends here, so this is the one place a run that ended
1178    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1179    /// notification centre. Best-effort: see [`crate::notices::raise`].
1180    pub async fn execute(&mut self) -> Result<()> {
1181        if let Some(selection) = &self.state.pr_recovery
1182            && (selection.source_proof.is_some() || selection.source_run.is_some())
1183        {
1184            Box::pin(selection.verify_source(&self.state.repo)).await?;
1185        }
1186        // Queue claims cover daemon work; this also serializes manual drivers
1187        // consuming a repair answer, including the new review and decision.
1188        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1189            || self.state.land_approval.is_some()
1190            || self.state.merge_decision.is_some()
1191            || !self.state.merge_repairs.is_empty()
1192            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1193        {
1194            let claim = FixClaim::acquire(&self.state.dir())?;
1195            let disk = RunState::load(&self.state.id)?;
1196            let checkpoint = |s: &RunState| {
1197                (
1198                    s.merge_repairs.len(),
1199                    s.merge_repairs
1200                        .last()
1201                        .map(|r| (r.question.clone(), r.phase)),
1202                    s.land_approval
1203                        .as_ref()
1204                        .map(|a| (a.question.clone(), a.head.clone())),
1205                )
1206            };
1207            if checkpoint(&disk) != checkpoint(&self.state) {
1208                bail!(
1209                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1210                );
1211            }
1212            Some(claim)
1213        } else {
1214            None
1215        };
1216        let mut execution = self.claim_driver()?;
1217        let runner = &mut *execution.runner;
1218        let wait = runner.repository_wait();
1219        // Keep the graph's large future off the callers' stack, including
1220        // standalone task attempts running on a default-sized test thread.
1221        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1222        let result = match result {
1223            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1224                record_repository_block(&mut runner.state, &e);
1225                runner.park_here()?;
1226                Ok(())
1227            }
1228            other => other,
1229        };
1230        execution.driver.finish(&mut runner.state)?;
1231        let ended = if result.is_err() {
1232            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1233        } else {
1234            crate::notices::run_ended(&runner.state)
1235        };
1236        if let Some(notice) = ended {
1237            crate::notices::raise(notice);
1238        }
1239        result
1240    }
1241
1242    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1243        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1244    }
1245
1246    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1247        let wait = self.repository_wait();
1248        coordinate_operation(&mut self.state, cwd, &wait).await
1249    }
1250
1251    /// Validate ownership and persist the driver claim in one critical section.
1252    fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1253        let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1254            Ok(owner) => owner,
1255            Err(error) => {
1256                let detail = format!("worktree maintenance or ownership update: {error:#}");
1257                // Admission did not own run.json: retain this resource outcome
1258                // in memory for queue settlement without overwriting a concurrent
1259                // retirement or publication transaction.
1260                self.state.resource_block = Some(crate::run::ResourceBlock {
1261                    node: "start".into(),
1262                    kind: "worktree maintenance".into(),
1263                    detail: detail.clone(),
1264                    at: Timestamp::now(),
1265                });
1266                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1267            }
1268        };
1269        let disk = RunState::load(&self.state.id)?;
1270        crate::pr_head::ensure_owner(&disk)?;
1271        if disk.liveness(false) == crate::run::Liveness::Live {
1272            bail!("run already has a live driver; wait for its execution to end");
1273        }
1274        if let Some(to) = &disk.released_to {
1275            bail!(
1276                "run {} cannot continue: its worktree was released to run {}",
1277                self.state.short(),
1278                crate::run::short_of(to)
1279            );
1280        }
1281        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1282        let mut execution = DriverExecution {
1283            runner: self,
1284            driver,
1285        };
1286        // Declare after the execution guard: on any claim error this short
1287        // lock must drop before that guard reloads and releases ownership.
1288        let _owner = owner;
1289        let state = &mut execution.runner.state;
1290        // Moving again, so it is no longer parked. Set before the walk rather
1291        // than in `resume`, so every way of re-entering the graph clears it
1292        // and a card cannot claim a run is waiting to be resumed while the
1293        // agents are already working.
1294        state.parked = false;
1295        state.resource_wait = None;
1296        state.resource_block = None;
1297        // Any seat this state still lists as answering belongs to whatever
1298        // process last drove this run — this one included, if it crashed
1299        // mid-wave. Cleared and flushed immediately, before anything else
1300        // runs, so a resume can never show a seat as live when nothing is
1301        // asking it anything yet; the node that actually dispatches the next
1302        // wave repopulates it.
1303        state.clear_active();
1304        // Recorded in the same spot, and flushed together with the clear
1305        // above: this is the pid a reader checks (`RunState::liveness`) when
1306        // no daemon claim exists to answer "is a process still driving this
1307        // run" — a plain `magi run` / `magi review` typed into a terminal
1308        // claims nothing there. Always overwritten, never only-if-absent, so
1309        // a resumed run's stale pid from a previous, possibly-dead process
1310        // can never survive into this one's own report. Unlike
1311        // `clear_active`, this changes on every single `execute()` call, so
1312        // the save below is now unconditional rather than only-if-cleared.
1313        //
1314        // `driver_started_at` is recorded in the same breath, from this same
1315        // pid, so `liveness` can tell a live pid that is genuinely still us
1316        // apart from one the OS has since handed to an unrelated process —
1317        // see that field's own doc for why the pid alone is not enough.
1318        let pid = std::process::id();
1319        state.driver_pid = Some(pid);
1320        state.driver_started_at = crate::proc::process_started_at(pid);
1321        state.driver_exited = false;
1322        state.begin_pass();
1323        execution.driver.bind(state);
1324        state.save()?;
1325        execution.driver.record(state)?;
1326        Ok(execution)
1327    }
1328
1329    async fn execute_graph(&mut self) -> Result<()> {
1330        self.state.configure_recovery_resume()?;
1331        // Preservation may invalidate an old empty classification. Resolve it
1332        // before any resume shortcut can reuse a verdict or publish a branch.
1333        self.persist_candidates().await?;
1334        if self
1335            .state
1336            .merge_decision
1337            .as_ref()
1338            .is_some_and(|decision| !decision.closed)
1339            && self.state.status != RunStatus::Landing
1340        {
1341            if crate::merge_decision::resumable_hold(&self.state) {
1342                self.state.merge = Some(MergeOutcome {
1343                    mode: MergeMode::Pr,
1344                    ok: false,
1345                    detail: self
1346                        .state
1347                        .merge_decision
1348                        .as_ref()
1349                        .unwrap()
1350                        .target
1351                        .pr
1352                        .clone(),
1353                    empty: false,
1354                });
1355                self.run_land().await?;
1356                self.settle_questions();
1357                return Ok(());
1358            }
1359            bail!(
1360                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1361            );
1362        }
1363        if self.state.config.graph.selected_implementer.is_some()
1364            && self.state.status == RunStatus::Blocked
1365            && self.state.tally.is_none()
1366            && self.state.candidate_preservation.is_empty()
1367            && self.state.interrupted_implementations.is_empty()
1368        {
1369            bail!(
1370                "selected implementation is held; explicitly release the owning task to retry it"
1371            );
1372        }
1373
1374        if self.state.merge_repairs.last().is_some_and(|r| {
1375            matches!(
1376                r.phase,
1377                crate::run::MergeRepairPhase::Requested
1378                    | crate::run::MergeRepairPhase::Fixing
1379                    | crate::run::MergeRepairPhase::Reviewing
1380            )
1381        }) {
1382            if self.repair_merge_review().await? {
1383                self.run_land().await?;
1384            }
1385            self.settle_questions();
1386            return Ok(());
1387        }
1388        // Publication already passed review and gate. Resume only that node:
1389        // syncing to a newer base here could change the verified commit.
1390        if self.state.pr_publication_pending {
1391            self.merge().await?;
1392            if self.resource_halt()? {
1393                return Ok(());
1394            }
1395            self.state.save()?;
1396            self.settle_questions();
1397            return Ok(());
1398        }
1399        // A run that already lost its quorum never resumes into the verdict
1400        // machinery: `deliberate` and `vote` would otherwise clobber the
1401        // stalled marker back to Voting and the run would keep going past a
1402        // verdict that is no longer trustworthy. Everything already recorded is
1403        // kept, so the run stays resumable (or foldable) for a human to pick up.
1404        //
1405        // On --resume the run gets one chance to repair itself: the seats a
1406        // rate limit took out are re-asked. If their quota has since reset and
1407        // the quorum is restored, the run picks up and finishes; otherwise it
1408        // stays stale and still-resumable for a later retry. If it does not
1409        // recover, the returned status stays `Stalled` and nothing was
1410        // clobbered (the recovery only mutates entries for the lost seats).
1411        if self.state.status == RunStatus::Stalled {
1412            if self.recover_stall().await? {
1413                self.finish_after_tally().await?;
1414            } else {
1415                // Still below quorum: persist the marker and stay resumable.
1416                self.state.save()?;
1417            }
1418            return Ok(());
1419        }
1420        // A run parked inside `land` - watching CI, mid fix-round, or
1421        // waiting on the owner's merge approval - resumes directly into it,
1422        // never back through `prep`. Everything before `merge` already
1423        // concluded; that is the only way `status` reaches `Landing` in the
1424        // first place. Re-walking `review_loop` first would also be actively
1425        // wrong: its own status recomputation (see its doc) treats any
1426        // clean round as reason to set `status` to `Gating`, which would
1427        // clobber this marker before `merge` ever ran, and this run would
1428        // never find its way back into `land` at all.
1429        if self.state.status == RunStatus::Landing
1430            || crate::handover::landing_checkpoint(&self.state)
1431        {
1432            self.state.land_sync_pending = true;
1433            self.state.save()?;
1434            self.run_land().await?;
1435            // `run_land` may have settled the run right here - CI came back
1436            // green and the PR merged, say - without ever passing back
1437            // through `merge`'s own trailing call. Whatever it left `status`
1438            // as is what this has to read.
1439            self.settle_questions();
1440            return Ok(());
1441        }
1442        // A run parked at `merge` on the owner's word about a withheld PR text
1443        // resumes straight into `merge`. Re-walking the tail would sync to the
1444        // base again (possibly rebasing, possibly spending the fixer) after the
1445        // gate had passed, and then push a tree nobody reviewed or gated.
1446        if self.state.github_text.is_some()
1447            && self.state.merge.is_none()
1448            && self.state.status == RunStatus::Gating
1449        {
1450            self.merge().await?;
1451            if self.resource_halt()? {
1452                return Ok(());
1453            }
1454            self.state.save()?;
1455            return Ok(());
1456        }
1457        if let Err(e) = self.prep().await {
1458            if e.is::<crate::repo_lock::ResourceBlocked>() {
1459                return Err(e);
1460            }
1461            self.state.status = RunStatus::Blocked;
1462            self.state
1463                .prep_failure
1464                .get_or_insert_with(|| format!("{e:#}"));
1465            self.state
1466                .event("prep", format!("preparation stopped: {e:#}"));
1467            self.state.save()?;
1468            return Err(e);
1469        }
1470        if self.park_here()? {
1471            return Ok(());
1472        }
1473        self.advise().await?;
1474        if self.park_here()? {
1475            return Ok(());
1476        }
1477        self.implement().await?;
1478        if self.park_here()? {
1479            return Ok(());
1480        }
1481        // `after_implement` already saved the state and settled any open
1482        // questions when it set this; nothing later in the graph has
1483        // anything to judge.
1484        if self.state.status == RunStatus::VerifiedNoop {
1485            return Ok(());
1486        }
1487        self.judge().await?;
1488        if self.park_here()? {
1489            return Ok(());
1490        }
1491        self.deliberate().await?;
1492        if self.park_here()? {
1493            return Ok(());
1494        }
1495        self.vote().await?;
1496        if self.park_here()? {
1497            return Ok(());
1498        }
1499        self.tally()?;
1500        // A verdict that lost its quorum is not trustworthy: do not review,
1501        // gate, or merge on it. Everything already done is kept, so the run
1502        // stays resumable (or foldable); the human can replace the agent that
1503        // ran out of quota and pick it up.
1504        if self.state.status == RunStatus::Stalled {
1505            // Persist the stalled marker now — the normal end-of-execute save
1506            // below is below this early return, and without it a resumed run
1507            // would reload a pre-tally status and keep going.
1508            self.state.save()?;
1509            return Ok(());
1510        }
1511        self.finish_after_tally().await?;
1512        Ok(())
1513    }
1514
1515    /// A wave could not start its agents because a shared resource stayed busy
1516    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1517    /// resumable. Unstarted seats have no result; completed peers retain
1518    /// their saved candidate state.
1519    fn stop_for_resource(&mut self) -> Result<()> {
1520        // A wait abandoned because the run is parking stays a park.
1521        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1522        {
1523            self.state.status = RunStatus::Blocked;
1524        }
1525        self.state.save()
1526    }
1527
1528    /// Stop walking the graph if a node ended on a resource block.
1529    fn resource_halt(&mut self) -> Result<bool> {
1530        if self.state.resource_block.is_none() {
1531            return Ok(false);
1532        }
1533        self.stop_for_resource()?;
1534        Ok(true)
1535    }
1536
1537    /// Park here if asked to, recording it in the run's own timeline.
1538    ///
1539    /// Returns whether the caller should stop walking the graph. The state is
1540    /// saved either way by the node that just finished; this adds the event so
1541    /// the operator's card says why a run that is neither finished nor moving
1542    /// is sitting where it is.
1543    fn park_here(&mut self) -> Result<bool> {
1544        let parking = self.pause.parked() || self.interrupt.parked();
1545        if !parking && self.state.resource_block.is_some() {
1546            self.stop_for_resource()?;
1547            return Ok(true);
1548        }
1549        // Either handle asking is enough - see `Pause`'s own doc for why
1550        // they are never the same one. `interrupt` is checked second so a
1551        // reason it carries is preferred in the message below over a plain
1552        // shutdown park racing it at the same boundary.
1553        if !self.pause.parked() && !self.interrupt.parked() {
1554            return Ok(false);
1555        }
1556        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1557        let why = match cause.as_ref().map(ParkCause::label) {
1558            Some(reason) => format!(
1559                "parked after `{}` ({reason}) — resume to carry on from here",
1560                self.state.status.as_str()
1561            ),
1562            None => format!(
1563                "parked after `{}` — resume to carry on from here",
1564                self.state.status.as_str()
1565            ),
1566        };
1567        self.state.event("park", why);
1568        self.state
1569            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1570        self.state.parked = true;
1571        self.state.save()?;
1572        Ok(true)
1573    }
1574
1575    /// Only a new explicit operator approval may retry failed preparation.
1576    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1577        if self.state.prep_failure.is_none() {
1578            return Ok(());
1579        }
1580        let failed_at = self
1581            .state
1582            .events
1583            .iter()
1584            .rev()
1585            .find(|event| event.node == "prep")
1586            .map_or(self.state.updated_at, |event| event.at);
1587        anyhow::ensure!(
1588            approval_at > failed_at,
1589            "preparation retry requires a new explicit release after the failure"
1590        );
1591        self.state.prep_failure = None;
1592        self.state.prep_complete = Some(false);
1593        self.state
1594            .event("prep-retry", "operator approved retry of preparation");
1595        Ok(())
1596    }
1597
1598    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1599    pub fn on_pause(&mut self, pause: Pause) {
1600        self.pause = pause;
1601    }
1602
1603    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1604    /// scheduler asking this one run - and no other - to park so a task
1605    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1606    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1607    /// handle.
1608    pub fn watch_interrupt(&mut self, pause: Pause) {
1609        self.interrupt = pause;
1610    }
1611
1612    /// Abandon this run's own open questions, once `status` has actually
1613    /// settled rather than merely paused.
1614    ///
1615    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1616    /// either back up with the candidates, the review round and the seat
1617    /// sessions already on disk, so a question an implementer asked mid-round
1618    /// may still get a real answer read by a real resume. Only the statuses
1619    /// `resumable` excludes are actually final: the run merged, it reached
1620    /// `Ready` with nothing left to do, it failed outright with no
1621    /// established point to continue from, or every candidate agreed, with
1622    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1623    /// every one of those the seat that asked is gone for good, exactly like
1624    /// the run being deleted under `magi run rm` - so the same cleanup
1625    /// applies, worded for what actually happened instead of "the run was
1626    /// deleted".
1627    ///
1628    /// Best-effort and silent on success: called from every place `status`
1629    /// can land on one of those three, including ones a resumed run revisits,
1630    /// so it must cost nothing when there was nothing open to begin with.
1631    fn settle_questions(&mut self) {
1632        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1633            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1634        }
1635    }
1636
1637    /// The tail of the graph after a trustworthy tally: fold losers, review,
1638    /// gate, merge, and persist.
1639    async fn finish_after_tally(&mut self) -> Result<()> {
1640        // Before the losers are folded: their branches are what the graft
1641        // reads from.
1642        self.graft().await?;
1643        if self.resource_halt()? {
1644            return Ok(());
1645        }
1646        self.fold_losers().await?;
1647        // Before review starts, and again right before the gate: a run's
1648        // review rounds can themselves take long enough for the base to move
1649        // a second time, and the gate is the one node whose "green" gets
1650        // acted on.
1651        self.sync_to_base().await?;
1652        if self.resource_halt()? {
1653            return Ok(());
1654        }
1655        if self.state.status == RunStatus::AlreadyInBase {
1656            return Ok(());
1657        }
1658        self.review_loop().await?;
1659        if self.resource_halt()? {
1660            return Ok(());
1661        }
1662        self.sync_to_base().await?;
1663        if self.resource_halt()? {
1664            return Ok(());
1665        }
1666        if self.state.status == RunStatus::AlreadyInBase {
1667            return Ok(());
1668        }
1669        self.gate().await?;
1670        self.merge().await?;
1671        if self.resource_halt()? {
1672            return Ok(());
1673        }
1674        self.state.save()?;
1675        Ok(())
1676    }
1677
1678    // ---------------------------------------------------------------- prep
1679
1680    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1681    /// run with the step and its output in the event; it is never swallowed.
1682    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1683    /// resume starts over instead of finding candidates and missing seats.
1684    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1685        let result = match ensure_setup_config(&mut self.state, repo).await {
1686            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1687            Err(e) => Err(e),
1688        };
1689        if let Err(e) = result {
1690            git::worktree_remove_clean(repo, wt).await.ok();
1691            if in_prep {
1692                self.rollback_prep(repo).await;
1693            }
1694            self.state.status = RunStatus::Blocked;
1695            self.state.event(
1696                "setup",
1697                format!("worktree setup failed in {}: {e:#}", wt.display()),
1698            );
1699            self.state.save()?;
1700            return Err(e);
1701        }
1702        Ok(())
1703    }
1704
1705    /// Undo everything `prep` created, so a resume re-runs it whole.
1706    async fn rollback_prep(&mut self, repo: &Path) {
1707        let root = self.state.worktree_root();
1708        for c in std::mem::take(&mut self.state.candidates) {
1709            if matches!(
1710                git::worktree_remove_clean(repo, &c.worktree).await,
1711                Ok(true)
1712            ) {
1713                git::branch_delete(repo, &c.branch).await.ok();
1714            } else {
1715                self.state.candidates.push(c);
1716            }
1717        }
1718        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1719        for j in 1..=seats {
1720            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1721                .await
1722                .ok();
1723        }
1724        for k in 1..=self.state.config.graph.advisors {
1725            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1726                .await
1727                .ok();
1728        }
1729    }
1730
1731    async fn prep(&mut self) -> Result<()> {
1732        if let Some(reason) = &self.state.prep_failure {
1733            bail!(
1734                "candidate preparation previously failed: {reason}; correct the source before retrying"
1735            );
1736        }
1737        if self
1738            .state
1739            .prep_complete
1740            .unwrap_or(!self.state.candidates.is_empty())
1741        {
1742            return Ok(());
1743        }
1744        let repo = self.state.repo.clone();
1745        // Includes rollback, config, refs, checkout creation and setup. No
1746        // repository mutation may precede the real shared lock acquisition.
1747        let _coordination = self.coordinate(&repo).await?;
1748        let base = self.state.base_commit.clone();
1749        let plan = self.state.sources.plan(&repo, &base).await?;
1750        self.rollback_prep(&repo).await;
1751        if !self.state.candidates.is_empty() {
1752            bail!(
1753                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1754            );
1755        }
1756        self.state.prep_complete = Some(false);
1757        self.state.status = RunStatus::Prep;
1758        self.state.save()?;
1759        let repo = self.state.repo.clone();
1760        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1761        let root = self.state.worktree_root();
1762        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1763
1764        // The hook is the write-time half of the blindness contract; the
1765        // presentation filter in `blind` is the half that cannot be bypassed.
1766        let hooks_dir = self.state.dir().join("hooks");
1767        if self.state.config.blind.commit_msg_hook {
1768            std::fs::create_dir_all(&hooks_dir)
1769                .with_context(|| format!("create {}", hooks_dir.display()))?;
1770            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1771            let path = hooks_dir.join("commit-msg");
1772            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1773            make_executable(&path)?;
1774            // Ref-counted rather than a plain idempotent set: with more than
1775            // one run able to be in flight in the same repository at once
1776            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1777            // true?" check cannot tell "another run of mine still needs
1778            // this" from "nobody does", and the run that happens to finish
1779            // first would disable the hook out from under a sibling still
1780            // relying on it.
1781            git::acquire_worktree_config(&repo).await?;
1782            self.state.enabled_worktree_config = true;
1783        }
1784
1785        // `[worktree] setup` hides its products per worktree, which needs the
1786        // same per-worktree config the hook does. Held until fold, like it.
1787        ensure_setup_config(&mut self.state, &repo).await?;
1788
1789        for (index, (spec, label)) in self
1790            .roles
1791            .implementers
1792            .clone()
1793            .into_iter()
1794            .zip(labels)
1795            .enumerate()
1796        {
1797            let branch = self.state.branch_for(label);
1798            let worktree = root.join(format!("cand-{label}"));
1799            // Persist intent before Git can leave a partially created tree.
1800            // Candidate registration follows setup, so it cannot inventory
1801            // crashes or refusals during creation itself.
1802            self.state.event(
1803                "prep-intent",
1804                serde_json::json!({
1805                    "branch": branch,
1806                    "worktree": worktree,
1807                    "start": start,
1808                    "picks": plan.picks,
1809                })
1810                .to_string(),
1811            );
1812            self.state.save()?;
1813            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1814            if self.state.config.blind.commit_msg_hook {
1815                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1816            }
1817            git::local_exclude(&worktree, "/.magi/").await?;
1818            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1819                // Nothing of this prep may survive: a resume must start over,
1820                // not find half the candidates and skip the rest.
1821                git::worktree_remove_clean(&repo, &worktree).await.ok();
1822                git::branch_delete(&repo, &branch).await.ok();
1823                self.rollback_prep(&repo).await;
1824                self.state.status = RunStatus::Blocked;
1825                self.state
1826                    .event("prep", format!("worktree setup failed: {e:#}"));
1827                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1828                self.state.save()?;
1829                return Err(e);
1830            }
1831            self.state.candidates.push(Candidate {
1832                index,
1833                label,
1834                agent: spec.id.clone(),
1835                branch,
1836                worktree,
1837                summary: String::new(),
1838                stat: String::new(),
1839                files: 0,
1840                commits: 0,
1841                empty: false,
1842                failed: None,
1843                verified_noop: None,
1844                duration_ms: 0,
1845                folded: false,
1846            });
1847            self.state.save()?;
1848            for pick in &plan.picks {
1849                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1850                {
1851                    self.state.status = RunStatus::Blocked;
1852                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1853                    self.state
1854                        .event("prep", format!("cannot apply referenced commit: {e}"));
1855                    self.state.save()?;
1856                    return Err(e);
1857                }
1858            }
1859        }
1860
1861        for j in 1..=self.roles.judges.len() {
1862            let wt = root.join(format!("judge-{j}"));
1863            if !wt.exists() {
1864                git::worktree_add_detached(&repo, &wt, &base).await?;
1865                self.setup_seat_worktree(&repo, &wt, true).await?;
1866            }
1867        }
1868
1869        // Disposable, detached checkouts for the design-deliberation stage's
1870        // advisor seats — the same shape as the judges' above, at the same
1871        // base commit, since advisors also only ever read. Sized off the
1872        // configured count directly rather than a resolved roster: unlike
1873        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1874        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1875        // `prep` has no `ResolvedRoles` field to read a count from here.
1876        if self.state.config.graph.advise {
1877            for k in 1..=self.state.config.graph.advisors {
1878                let wt = root.join(format!("advisor-{k}"));
1879                if !wt.exists() {
1880                    git::worktree_add_detached(&repo, &wt, &base).await?;
1881                    self.setup_seat_worktree(&repo, &wt, true).await?;
1882                }
1883            }
1884        }
1885
1886        // A judge cannot tell it is looking at its own patch — the seats keep
1887        // separate conversations — but a panel that shares agents with the
1888        // field is less independent than it looks, and that is worth saying out
1889        // loud once per run rather than leaving it in the config.
1890        let authors: Vec<&str> = self
1891            .roles
1892            .implementers
1893            .iter()
1894            .map(|a| a.id.as_str())
1895            .collect();
1896        let overlap: Vec<String> = self
1897            .roles
1898            .judges
1899            .iter()
1900            .enumerate()
1901            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1902            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1903            .collect();
1904        if !overlap.is_empty() {
1905            let note = format!(
1906                "{} also authored a candidate; blind, but the panel is less \
1907                 independent than {} distinct agents would be",
1908                overlap.join(", "),
1909                self.roles.judges.len()
1910            );
1911            self.state.event("prep", note);
1912        }
1913
1914        self.state.event(
1915            "prep",
1916            format!(
1917                "{} candidates, {} judges, base {} ({})",
1918                self.state.candidates.len(),
1919                self.roles.judges.len(),
1920                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1921                self.state.base_branch
1922            ),
1923        );
1924        self.state.prep_complete = Some(true);
1925        self.state.status = RunStatus::Implementing;
1926        self.state.save()?;
1927        Ok(())
1928    }
1929
1930    // -------------------------------------------------------------- advise
1931
1932    /// The design-deliberation stage: independent, read-only advisor seats
1933    /// each sketch a design before any implementer touches the repository,
1934    /// and (when at least one produced a usable proposal) a synthesis seat
1935    /// blends them into a brief `implement` carries in every candidate's
1936    /// prompt.
1937    ///
1938    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1939    /// advisors` is the proposal count. Everything here is best-effort and
1940    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1941    /// that cannot reach quota, or a synthesis seat that produced nothing
1942    /// usable all leave `implement` exactly as it was before this stage
1943    /// existed — the task instruction alone — rather than failing the whole
1944    /// competition over an enrichment stage. Every outcome is still recorded
1945    /// as an event, so a run that got nothing from this stage says why.
1946    ///
1947    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1948    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1949    /// resumed run whose stage failed would re-run it, and re-spend the
1950    /// agent calls, on every reentry before `implement`.
1951    ///
1952    /// Also skipped once any candidate shows implementation progress — the
1953    /// exact predicate `implement` itself uses to decide a candidate is no
1954    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1955    /// is not enough: a run created by an older binary that predates this
1956    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1957    /// an already-`Implementing`-or-later run under this build would
1958    /// otherwise walk straight back through `prep` (a no-op once candidates
1959    /// exist) into this node and spawn every advisor seat against worktrees
1960    /// `prep` never recreated — after implementation has already started,
1961    /// which is exactly the invariant this stage exists to guarantee.
1962    async fn advise(&mut self) -> Result<()> {
1963        let implement_untouched = self
1964            .state
1965            .candidates
1966            .iter()
1967            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1968        if !self.state.config.graph.advise || self.state.advise_attempted {
1969            return Ok(());
1970        }
1971        if !implement_untouched {
1972            self.state.event(
1973                "advise",
1974                "skipping the design-deliberation stage: at least one \
1975                 candidate already shows implementation progress, so this \
1976                 run is past the point the stage exists to run before"
1977                    .to_owned(),
1978            );
1979            self.state.advise_attempted = true;
1980            self.state.save()?;
1981            return Ok(());
1982        }
1983        let run_id = self.state.id.clone();
1984        let prompts = self.state.config.prompts.clone();
1985        let instruction = self.state.instruction.clone();
1986        let language = self.state.config.graph.language.clone();
1987        let root = self.state.worktree_root();
1988        let n = self.state.config.graph.advisors;
1989        let where_recorded = self.state.dir().join("run.json");
1990
1991        let seats = match self.state.config.advisors() {
1992            Ok(seats) if !seats.is_empty() => seats,
1993            Ok(_) => {
1994                self.state.event(
1995                    "advise",
1996                    format!(
1997                        "[graph] advisors is 0; skipping the design-deliberation \
1998                         stage and continuing without a synthesis brief (see {})",
1999                        where_recorded.display()
2000                    ),
2001                );
2002                self.state.advise_attempted = true;
2003                self.state.save()?;
2004                return Ok(());
2005            }
2006            Err(e) => {
2007                self.state.event(
2008                    "advise",
2009                    format!(
2010                        "could not resolve advisor seats ({e:#}); continuing \
2011                         without a design-deliberation brief (see {})",
2012                        where_recorded.display()
2013                    ),
2014                );
2015                self.state.advise_attempted = true;
2016                self.state.save()?;
2017                return Ok(());
2018            }
2019        };
2020
2021        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2022        let artifacts = agent::artifacts_dir(&self.state.dir());
2023        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2024
2025        let mut jobs = Vec::new();
2026        for (i, spec) in seats.iter().cloned().enumerate() {
2027            let seat_key = format!("advisor-{}", i + 1);
2028            let seat = self.seat(&seat_key, &spec.id);
2029            jobs.push(SeatJob {
2030                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2031                spec,
2032                seat,
2033                cwd: worktrees[i % worktrees.len()].clone(),
2034                timeout,
2035                allow_write: false,
2036                sessions: false,
2037                artifacts: artifacts.clone(),
2038                stem: seat_key,
2039                handover: None,
2040            });
2041        }
2042
2043        self.state.event(
2044            "advise",
2045            format!(
2046                "{} advisor seat(s) sketching a design in parallel",
2047                jobs.len()
2048            ),
2049        );
2050        let mut quota_losses = Vec::new();
2051        let cache = self.state.config.cache_dir();
2052        let ctx = WaveCtx {
2053            carry_seats: false,
2054            pauses: Some((&self.pause, &self.interrupt)),
2055            run: &run_id,
2056            node: "advise",
2057            prompts: &prompts,
2058            cache: cache.as_deref(),
2059            round: None,
2060        };
2061        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2062        let results = ask_json_wave::<Proposal>(
2063            jobs,
2064            Arc::clone(&self.sem),
2065            self.state.config.graph.retries,
2066            &advisor_roster,
2067            &ctx,
2068            &mut quota_losses,
2069            &mut self.state,
2070            &|p: &Proposal| p.validate(),
2071        )
2072        .await;
2073        if self.state.resource_block.is_some() {
2074            return self.stop_for_resource();
2075        }
2076        self.state.quota.extend(quota_losses);
2077
2078        let mut records = Vec::with_capacity(results.len());
2079        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2080            let agent_id = seat.agent.clone();
2081            self.state.seats.insert(seat.key.clone(), seat);
2082            match res {
2083                Ok((proposal, out)) => {
2084                    self.state
2085                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2086                    records.push(advise::AdvisorRecord::proposed(
2087                        i + 1,
2088                        agent_id,
2089                        proposal,
2090                        out.duration_ms,
2091                    ));
2092                }
2093                Err(e) => {
2094                    self.state.event(
2095                        "advise",
2096                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2097                    );
2098                    records.push(advise::AdvisorRecord::failed(
2099                        i + 1,
2100                        agent_id,
2101                        e.to_string(),
2102                    ));
2103                }
2104            }
2105        }
2106
2107        let mut advice = advise::Advice {
2108            records,
2109            synthesis: None,
2110        };
2111        if advice.proposals().is_empty() {
2112            self.state.event(
2113                "advise",
2114                "no advisor produced a usable proposal; continuing without a \
2115                 synthesis brief"
2116                    .to_owned(),
2117            );
2118        } else {
2119            match self
2120                .synthesize_brief(
2121                    &advice,
2122                    &instruction,
2123                    &language,
2124                    &worktrees[0],
2125                    &artifacts,
2126                    &run_id,
2127                    &prompts,
2128                    cache.as_deref(),
2129                )
2130                .await
2131            {
2132                Ok(Some(text)) => {
2133                    self.state.event(
2134                        "advise",
2135                        "synthesized a design brief for the implementer".to_owned(),
2136                    );
2137                    advice.synthesis = Some(text);
2138                }
2139                Ok(None) => {
2140                    self.state.event(
2141                        "advise",
2142                        "the synthesis seat produced nothing usable; continuing \
2143                         without a design brief"
2144                            .to_owned(),
2145                    );
2146                }
2147                Err(e) => {
2148                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2149                        return Err(e);
2150                    }
2151                    self.state.event(
2152                        "advise",
2153                        format!("could not synthesize a design brief: {e:#}"),
2154                    );
2155                }
2156            }
2157        }
2158        advise::apply_reflection(&mut advice);
2159
2160        self.state.advice = Some(advice);
2161        self.state.advise_attempted = true;
2162        self.state.save()?;
2163        Ok(())
2164    }
2165
2166    /// The synthesis seat: reads every advisor's proposal and blends them
2167    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2168    /// out of [`Runner::advise`] only for readability — it is not called
2169    /// anywhere else.
2170    ///
2171    /// Picked the same way [`crate::talk`]'s standing conversation and
2172    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2173    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2174    /// order (a claude seat, else the first runnable agent in roster order)
2175    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2176    /// in [`crate::config`] for why a dedicated field exists here at all.
2177    #[allow(clippy::too_many_arguments)]
2178    async fn synthesize_brief(
2179        &mut self,
2180        advice: &advise::Advice,
2181        instruction: &str,
2182        language: &str,
2183        cwd: &Path,
2184        artifacts: &Path,
2185        run_id: &str,
2186        prompts: &Prompts,
2187        cache: Option<&Path>,
2188    ) -> Result<Option<String>> {
2189        let chain = agent::pick_chain(
2190            &self.state.config.agents,
2191            self.state.config.roles.synthesizer.as_ref(),
2192            &agent::installed,
2193            "synthesizer",
2194        )?;
2195        let proposals = advice.proposals();
2196        let mut prompt = prompt::with_overlay(
2197            prompt::synthesize_brief(instruction, &proposals, language),
2198            prompts.overlay("advise"),
2199        );
2200        if cache.is_some() {
2201            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2202            // below — see `prompt::build_cache_note`'s doc for why telling a
2203            // read-only seat to build through the shared cache is exactly how
2204            // a sandbox's write refusal gets misread as a defect.
2205            prompt.push('\n');
2206            prompt.push_str(&prompt::build_cache_note("advise", false));
2207        }
2208        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2209        // Each id is tried once, in order; a quota hit, error or unusable
2210        // answer moves to the next. The seat is single-turn (`sessions:
2211        // false`) and the prompt is the whole context, so a fallback agent
2212        // needs nothing carried over.
2213        let mut last = None;
2214        for (n, spec) in chain.iter().enumerate() {
2215            if n > 0 {
2216                self.state
2217                    .event("advise", format!("synthesis falling back to {}", spec.id));
2218            }
2219            let mut seat = self.seat("advise-synthesis", &spec.id);
2220            let wait = self.repository_wait();
2221            let outcome = wait
2222                .scope(agent::invoke(
2223                    spec,
2224                    &mut seat,
2225                    &Invocation {
2226                        scope: crate::agent::InvocationScope::Repository,
2227                        coordination: self.state.config.coordination.as_ref(),
2228                        fetched_base: None,
2229                        cwd,
2230                        prompt: &prompt,
2231                        timeout,
2232                        allow_write: false,
2233                        unsandboxed: false,
2234                        sessions: false,
2235                        artifacts,
2236                        stem: &if n == 0 {
2237                            "advise-synthesis".to_owned()
2238                        } else {
2239                            format!("advise-synthesis-{}", spec.id)
2240                        },
2241                        run: run_id,
2242                        node: "advise",
2243                        cache_dir: None,
2244                        attachments: &[],
2245                        writable: &[],
2246                    },
2247                ))
2248                .await;
2249            if outcome.is_ok() {
2250                self.state.seats.insert(seat.key.clone(), seat);
2251            }
2252            let advance = agent::chain_advances(&outcome);
2253            last = Some(outcome);
2254            if !advance {
2255                break;
2256            }
2257        }
2258        // Exhausted: the last attempt's result is what a single failed seat
2259        // would have produced.
2260        let out = last.expect("a chain holds at least one agent")?;
2261        if !out.usable() {
2262            return Ok(None);
2263        }
2264        let text =
2265            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2266        Ok((!text.trim().is_empty()).then_some(text))
2267    }
2268
2269    // ----------------------------------------------------------- implement
2270
2271    async fn implement(&mut self) -> Result<()> {
2272        self.persist_candidates().await?;
2273        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2274        // agent files with `magi task add` name the run that paid for it. The
2275        // prompt overlay is cloned alongside it because the waves borrow it
2276        // while `self` is mutably borrowed by the node's own bookkeeping.
2277        let run_id = self.state.id.clone();
2278        let prompts = self.state.config.prompts.clone();
2279        let todo: Vec<usize> = self
2280            .state
2281            .candidates
2282            .iter()
2283            .enumerate()
2284            .filter(|(_, c)| {
2285                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2286                    && c.failed.is_none()
2287                    && !c.empty
2288            })
2289            .map(|(i, _)| i)
2290            .collect();
2291        if todo.is_empty() {
2292            return self.after_implement();
2293        }
2294        self.state.status = RunStatus::Implementing;
2295
2296        let language = self.state.config.graph.language.clone();
2297        let github_language = self.state.config.graph.github_language.clone();
2298        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2299        let sessions = self.state.config.graph.sessions;
2300        let artifacts = agent::artifacts_dir(&self.state.dir());
2301        // The design-deliberation stage's blended brief, when `advise` found
2302        // one — carried into every implementer's prompt the same way
2303        // regardless of which candidate it is.
2304        let brief = self
2305            .state
2306            .advice
2307            .as_ref()
2308            .and_then(|a| a.synthesis.as_deref())
2309            .map(str::to_owned);
2310        let attachments = self.state.attachments.clone();
2311
2312        let mut jobs = Vec::new();
2313        for &i in &todo {
2314            let (index, label, worktree) = {
2315                let c = &self.state.candidates[i];
2316                (c.index, c.label, c.worktree.clone())
2317            };
2318            let spec = self.roles.implementers[index].clone();
2319            let seat_key = format!("impl-{label}");
2320            let seat = self.seat(&seat_key, &spec.id);
2321            let instruction = seeded_instruction(&self.state);
2322            jobs.push(SeatJob {
2323                spec,
2324                seat,
2325                prompt: prompt::implement_in(
2326                    &instruction,
2327                    &worktree.to_string_lossy(),
2328                    &language,
2329                    &github_language,
2330                    brief.as_deref(),
2331                    &attachments,
2332                ),
2333                cwd: worktree,
2334                timeout,
2335                allow_write: true,
2336                sessions,
2337                artifacts: artifacts.clone(),
2338                stem: format!("impl-{label}"),
2339                handover: None,
2340            });
2341        }
2342
2343        self.state.event(
2344            "implement",
2345            format!("{} candidates in parallel", jobs.len()),
2346        );
2347        // Kept so a seat whose CLI hung up can be asked again from the same
2348        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2349        // can update a seat's own entry once a fallback agent takes it over —
2350        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2351        // whichever agent actually answered, not the one that quota'd out.
2352        let mut sent = jobs.clone();
2353        let cache = self.state.config.cache_dir();
2354        let ctx = WaveCtx {
2355            carry_seats: false,
2356            pauses: Some((&self.pause, &self.interrupt)),
2357            run: &run_id,
2358            node: "implement",
2359            prompts: &prompts,
2360            cache: cache.as_deref(),
2361            round: None,
2362        };
2363        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2364        if !wave_blocked(&results) {
2365            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2366                .await;
2367            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2368                .await?;
2369            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2370                .await;
2371        }
2372        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2373
2374        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2375            && results
2376                .iter()
2377                .any(|(_, _, out)| FailClass::of(out).is_some());
2378        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2379            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2380                // Executed turns interrupted by disk pressure already carry
2381                // a preservation barrier from wave. Admission-only waits do
2382                // not. Keep the interrupted seat's session for recovery too.
2383                if self
2384                    .state
2385                    .candidate_preservation
2386                    .contains_key(&self.state.candidates[i].label)
2387                {
2388                    self.state.seats.insert(seat.key.clone(), seat);
2389                }
2390                continue;
2391            }
2392            let seat_key = seat.key.clone();
2393            // A quota fallback (`resume_seat_handovers`) may have handed this
2394            // seat to a different agent than the one `prep` recorded on the
2395            // candidate; the stats tables and any later fixer-defaults-to-
2396            // winner's-author lookup must credit whoever actually answered —
2397            // unless every fallback also quota'd out, in which case nobody
2398            // actually answered and crediting the last agent tried would
2399            // erase every earlier agent's own quota loss from the stats
2400            // tables instead of just this one seat's.
2401            let agent = seat.agent.clone();
2402            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2403            self.state.seats.insert(seat.key.clone(), seat);
2404            let label = self.state.candidates[i].label;
2405            self.state.interrupted_implementations.remove(&label);
2406
2407            let (summary, duration, failed, verified_claim) = match out {
2408                AgentOutcome::Ok(o) => {
2409                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2410                    let failed = (!o.usable()).then(|| {
2411                        if o.timed_out {
2412                            "agent timed out".to_owned()
2413                        } else {
2414                            format!(
2415                                "agent exited with {}",
2416                                crate::notices::exit_status(o.exit_code)
2417                            )
2418                        }
2419                    });
2420                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2421                    (text, o.duration_ms, failed, verified_claim)
2422                }
2423                // Left un-resumed by `resume_undelivered` (a dirty tree
2424                // already rescues the work, or there was no session left to
2425                // resume into) — reported like the ordinary failure it is,
2426                // never as if `o.text` (the CLI's raw error JSON) were an
2427                // answer.
2428                AgentOutcome::Dropped(o) => {
2429                    let why = o
2430                        .dropped
2431                        .as_ref()
2432                        .map(|d| d.why.as_str())
2433                        .unwrap_or("the CLI ended the stream without delivering its answer");
2434                    (
2435                        String::new(),
2436                        o.duration_ms,
2437                        Some(format!("the CLI dropped the stream ({why})")),
2438                        None,
2439                    )
2440                }
2441                AgentOutcome::Quota(o) => {
2442                    self.state.quota.push(QuotaLoss {
2443                        seat: seat_key,
2444                        node: "implement".to_owned(),
2445                        at: Timestamp::now(),
2446                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2447                    });
2448                    (
2449                        String::new(),
2450                        o.duration_ms,
2451                        Some("rate limited (quota); produced no change".to_owned()),
2452                        None,
2453                    )
2454                }
2455                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2456                    (String::new(), 0, Some(e), None)
2457                }
2458            };
2459
2460            // Save every completed turn before any fallible Git operation. A failed
2461            // rescue must not erase the reply or prevent collection of other seats.
2462            let c = &mut self.state.candidates[i];
2463            if !exhausted_the_fallback_chain {
2464                c.agent = agent;
2465            }
2466            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2467            c.duration_ms = duration;
2468            c.failed = failed;
2469            c.verified_noop = verified_claim;
2470            self.state
2471                .candidate_preservation
2472                .insert(label, "awaiting persistence".to_owned());
2473        }
2474        self.state.save()?;
2475        self.persist_candidates().await?;
2476
2477        if resource_blocked {
2478            return self.stop_for_resource();
2479        }
2480        if selected_failed {
2481            self.state.status = RunStatus::Blocked;
2482            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2483            self.state.save()?;
2484            bail!(
2485                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2486            );
2487        }
2488        self.after_implement()
2489    }
2490
2491    /// Retry only persistence after a completed implementation turn.
2492    async fn persist_candidates(&mut self) -> Result<()> {
2493        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2494        for label in labels {
2495            let i = self
2496                .state
2497                .candidates
2498                .iter()
2499                .position(|c| c.label == label)
2500                .context("pending candidate is missing")?;
2501            let c = self.state.candidates[i].clone();
2502            let interrupted = self.state.interrupted_implementations.contains(&label);
2503            let result = async {
2504                let wait = crate::repo_lock::WaitPolicy::new(
2505                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2506                    || false,
2507                );
2508                let _coordination =
2509                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2510                // Best effort evidence requires no index writes. A successful snapshot
2511                // never authorizes removal of a tree whose rescue failed.
2512                if let Err(e) = git::snapshot_uncommitted(
2513                    &c.worktree,
2514                    &self
2515                        .state
2516                        .dir()
2517                        .join("artifacts")
2518                        .join(format!("candidate-{label}-recovery")),
2519                )
2520                .await
2521                {
2522                    self.state.event(
2523                        "preserve",
2524                        format!("candidate {label} snapshot incomplete: {e:#}"),
2525                    );
2526                }
2527                let rescued = git::rescue_commit(
2528                    &c.worktree,
2529                    &format!("magi: candidate {label} (uncommitted work)"),
2530                )
2531                .await?;
2532                self.state.note_withheld("implement", &rescued.withheld);
2533                let base = &self.state.base_commit;
2534                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2535                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2536                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2537                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2538                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2539                if c.empty && commits > 0 && !patch.trim().is_empty()
2540                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2541                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2542                        || self.state.tally.is_some())
2543                {
2544                    // Keep the old classification and pending marker so every
2545                    // retry stops too. Retain decisions as recovery evidence;
2546                    // only a supported recovery operation may replace them.
2547                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2548                }
2549                let c = &mut self.state.candidates[i];
2550                c.commits = commits;
2551                c.stat = stat;
2552                c.files = files;
2553                let no_changes = commits == 0 || patch.trim().is_empty();
2554                c.empty = no_changes && !interrupted;
2555                if interrupted && no_changes {
2556                    c.failed = None;
2557                    c.verified_noop = None;
2558                }
2559                if !c.empty {
2560                    c.failed = None;
2561                    c.verified_noop = None;
2562                }
2563                self.state.event(
2564                    "preserve",
2565                    format!("candidate {label}: {files} files, {commits} commits"),
2566                );
2567                Ok::<_, anyhow::Error>(())
2568            }
2569            .await;
2570            match result {
2571                Ok(()) => {
2572                    self.state.candidate_preservation.remove(&label);
2573                    if !interrupted || self.state.candidates[i].files > 0 {
2574                        self.state.interrupted_implementations.remove(&label);
2575                    }
2576                }
2577                Err(e) => {
2578                    let reason = format!("candidate {label} preservation failed: {e:#}");
2579                    self.state.event("preserve", &reason);
2580                    self.state.candidate_preservation.insert(label, reason);
2581                }
2582            }
2583            if self.state.candidate_preservation.is_empty()
2584                && self.state.status == RunStatus::Blocked
2585                && self.state.candidates.iter().all(|c| c.failed.is_none())
2586            {
2587                // Persist the recovered phase with the cleared pending marker;
2588                // a crash between saves must not turn this into a selected-seat
2589                // failure hold on the next resume.
2590                self.state.status = RunStatus::Implementing;
2591                self.state.event(
2592                    "preserve",
2593                    "candidate persistence recovered; implementation can continue",
2594                );
2595            }
2596            self.state.save()?;
2597        }
2598        if !self.state.candidate_preservation.is_empty() {
2599            self.state.status = RunStatus::Blocked;
2600            self.state.save()?;
2601            bail!(
2602                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2603            );
2604        }
2605        Ok(())
2606    }
2607
2608    /// Ask again, once, for work a CLI did and then failed to hand over.
2609    ///
2610    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2611    /// status with an empty response and a usage report showing output tokens,
2612    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2613    /// seven minutes and 14,267 output tokens that arrived as an empty
2614    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2615    ///
2616    /// Two conditions, and both matter:
2617    ///
2618    /// - **Only when the tree is untouched.** Often the agent has already
2619    ///   written its files and only the closing message was lost; the rescue
2620    ///   commit below picks that up and there is nothing to ask for. Re-asking
2621    ///   then would pay for a second implementation of work already on disk.
2622    /// - **Once.** A CLI that drops one stream can drop the next, and this
2623    ///   node is the most expensive in the graph.
2624    ///
2625    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2626    /// dropped reply still carried its `conversation_id`, so the seat is asked
2627    /// to finish what it was doing rather than sent the whole task again. It
2628    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2629    /// node's - for the same reason a re-ranked judge does: restating finished
2630    /// work is not the work.
2631    ///
2632    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2633    /// same way until it resets, while an abandoned conversation is still
2634    /// there to be picked up.
2635    async fn resume_undelivered(
2636        &mut self,
2637        results: &mut [(usize, SeatState, AgentOutcome)],
2638        sent: &[SeatJob],
2639        prompts: &Prompts,
2640        run_id: &str,
2641    ) {
2642        for (wi, seat, out) in results.iter_mut() {
2643            let Some(dropped) = (match &*out {
2644                AgentOutcome::Dropped(o) => o.dropped.clone(),
2645                _ => None,
2646            }) else {
2647                continue;
2648            };
2649            let Some(job) = sent.get(*wi) else { continue };
2650            // Already on disk? Then only the closing message was lost.
2651            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2652                self.state.event(
2653                    "implement",
2654                    format!(
2655                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2656                         work is in the tree",
2657                        seat.key, dropped.output_tokens, dropped.why
2658                    ),
2659                );
2660                continue;
2661            }
2662            // The re-ask only makes sense as a resume: `resume_after_drop`
2663            // says nothing about the task, trusting the seat to still hold it.
2664            // Without a session to resume — sessions disabled, or this CLI's
2665            // drop shape happened not to carry a session id — that prompt
2666            // would open a brand-new conversation with no context at all,
2667            // which is worse than leaving this as the ordinary failure it
2668            // already is.
2669            if !has_context(&job.spec, seat, job.sessions) {
2670                self.state.event(
2671                    "implement",
2672                    format!(
2673                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2674                         is no session left to resume",
2675                        seat.key, dropped.output_tokens, dropped.why
2676                    ),
2677                );
2678                continue;
2679            }
2680            self.state.event(
2681                "implement",
2682                format!(
2683                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2684                     conversation",
2685                    seat.key, dropped.output_tokens, dropped.why
2686                ),
2687            );
2688            let mut retry = job.clone();
2689            retry.seat = seat.clone();
2690            retry.prompt = prompt::resume_after_drop(&dropped.why);
2691            retry.timeout = retry_budget(job.timeout, true);
2692            retry.stem = format!("{}-resume", job.stem);
2693            let cache = self.state.config.cache_dir();
2694            let ctx = WaveCtx {
2695                carry_seats: false,
2696                pauses: Some((&self.pause, &self.interrupt)),
2697                run: run_id,
2698                node: "implement",
2699                prompts,
2700                cache: cache.as_deref(),
2701                round: None,
2702            };
2703            let (resumed_seat, resumed) =
2704                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2705            *seat = resumed_seat;
2706            *out = resumed;
2707        }
2708    }
2709
2710    /// Fall an implement seat through to the next untried agent in the
2711    /// implementer roster when it lost to quota — or, since the handover was
2712    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2713    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2714    /// quota path itself being unchanged — instead of leaving the
2715    /// seat's loss final the moment one agent's account runs dry.
2716    ///
2717    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2718    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2719    /// truncates to the single slot rotation picked, so a solo task whose one
2720    /// implementer hits quota mid-run used to have nothing else to try. This
2721    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2722    /// unrotated roster — which is the only place the *other* candidates in
2723    /// the machine's roster still exist once `implementers` has been cut down
2724    /// to size.
2725    ///
2726    /// Walks forward from just past the seat's own original position in the
2727    /// roster, never wrapping back to the front: a later candidate slot (say
2728    /// `beta`, the roster's second entry) must fall through to the *next*
2729    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2730    /// almost certainly a different candidate's own agent already — and once
2731    /// the roster's tail is exhausted there is nothing left to fall through
2732    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2733    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2734    /// let this retry that id forever. The loop keeps falling through until
2735    /// an attempt lands something other than `Quota` or the roster's tail
2736    /// runs out of untried ids, at which point the seat is left exactly as
2737    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2738    /// `QuotaLoss` recorded, the candidate failed/empty.
2739    ///
2740    /// `sent` is taken mutably and updated with the fallback agent's spec:
2741    /// `resume_unconfirmed_commands`, which runs after this and also reads
2742    /// `sent`, must see whichever agent actually ended up answering the seat
2743    /// — reading the stale, original spec there would check session
2744    /// eligibility against the wrong CLI and could hand a fallback agent's
2745    /// session id to the agent that just lost the seat to quota.
2746    ///
2747    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2748    /// — `self.seat` only reuses state when the agent id is unchanged, so
2749    /// handing it a different id already gets this for free. Reusing the old
2750    /// seat would resume a different CLI's session as if it were a
2751    /// continuation of this one.
2752    ///
2753    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2754    /// a quota loss cuts an agent off mid-turn, so anything already in the
2755    /// tree is unfinished work, not a completed candidate a re-ask would pay
2756    /// for twice. A dirty tree is rescued into a commit first (the same
2757    /// neutral-identity rescue `implement`'s own outcome loop gives every
2758    /// candidate) so the next agent starts clean.
2759    ///
2760    /// The new agent gets the implementer's full prompt and full
2761    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2762    /// it has no session and no context, and is implementing the task from
2763    /// nothing, unlike a resumed drop which is only restating work already
2764    /// done.
2765    ///
2766    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2767    /// `implement` event, never into `self.state.quota` — that is what
2768    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2769    /// go unspent, and a seat that ultimately recovered on its second or
2770    /// third agent is not the stalled panel that check exists to catch. Only
2771    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2772    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2773    /// outcome loop already has — this helper never pushes to it itself.
2774    async fn resume_seat_handovers(
2775        &mut self,
2776        results: &mut [(usize, SeatState, AgentOutcome)],
2777        sent: &mut [SeatJob],
2778        prompts: &Prompts,
2779        run_id: &str,
2780    ) -> Result<()> {
2781        let instruction = seeded_instruction(&self.state);
2782        let language = self.state.config.graph.language.clone();
2783        let github_language = self.state.config.graph.github_language.clone();
2784        let brief = self
2785            .state
2786            .advice
2787            .as_ref()
2788            .and_then(|a| a.synthesis.as_deref())
2789            .map(str::to_owned);
2790        let attachments = self.state.attachments.clone();
2791        for (wi, seat, out) in results.iter_mut() {
2792            // Who holds the other candidate seats of this wave right now
2793            // (earlier handovers already written back to `sent`).
2794            let others: BTreeSet<String> = sent
2795                .iter()
2796                .enumerate()
2797                .filter(|(j, _)| j != wi)
2798                .map(|(_, j)| j.spec.id.clone())
2799                .collect();
2800            let Some(job) = sent.get_mut(*wi) else {
2801                continue;
2802            };
2803            // Where the seat's own original agent sits in the roster — the
2804            // fallback walk starts just past here, never at the front, so a
2805            // later candidate slot's quota loss does not fall back onto an
2806            // earlier slot's own agent.
2807            let start = self
2808                .roles
2809                .implementer_roster
2810                .iter()
2811                .position(|s| s.id == job.spec.id)
2812                .unwrap_or(0);
2813            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2814            let mut fallback_attempt = 0usize;
2815            let mut prev: Option<FailClass> = None;
2816            while let Some(cur) = FailClass::of(&*out) {
2817                if !should_hand_over(prev.as_ref(), &cur) {
2818                    break;
2819                }
2820                let Some(next) =
2821                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2822                        .cloned()
2823                else {
2824                    break;
2825                };
2826                tried.insert(next.id.clone());
2827                fallback_attempt += 1;
2828
2829                let coordination = self.coordinate(&job.cwd).await?;
2830                if let Ok(r) = git::rescue_commit(
2831                    &job.cwd,
2832                    &format!(
2833                        "magi: candidate {} (uncommitted work before {} fallback)",
2834                        seat.key,
2835                        if cur == FailClass::Quota {
2836                            "quota"
2837                        } else {
2838                            "handover"
2839                        }
2840                    ),
2841                )
2842                .await
2843                {
2844                    self.state.note_withheld("implement", &r.withheld);
2845                }
2846                drop(coordination);
2847
2848                record_handover(
2849                    &mut self.state,
2850                    "implement",
2851                    &seat.key,
2852                    &seat.agent,
2853                    &next.id,
2854                    &cur,
2855                    &fail_reason(&*out),
2856                );
2857                prev = Some(cur.clone());
2858
2859                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2860                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2861                // Kept in sync on `sent` itself, not just the local retry: a
2862                // later helper (`resume_unconfirmed_commands`) reads `sent`
2863                // after this one returns and must see whichever agent is now
2864                // occupying the seat, not the one that just quota'd out —
2865                // otherwise it would judge session/continuation eligibility
2866                // by the wrong CLI and could resend a fallback's session id
2867                // to the agent that lost it the seat in the first place.
2868                job.spec = next.clone();
2869                let mut retry = job.clone();
2870                retry.seat = new_seat;
2871                retry.prompt = prompt::implement_in(
2872                    &instruction,
2873                    &job.cwd.to_string_lossy(),
2874                    &language,
2875                    &github_language,
2876                    brief.as_deref(),
2877                    &attachments,
2878                );
2879                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2880                let cache = self.state.config.cache_dir();
2881                let ctx = WaveCtx {
2882                    carry_seats: false,
2883                    pauses: Some((&self.pause, &self.interrupt)),
2884                    run: run_id,
2885                    node: "implement",
2886                    prompts,
2887                    cache: cache.as_deref(),
2888                    round: None,
2889                };
2890                let (fallback_seat, fallback_out) = run_one(
2891                    retry,
2892                    Arc::clone(&self.sem),
2893                    &ctx,
2894                    &mut self.state,
2895                    fallback_attempt,
2896                )
2897                .await;
2898                *seat = fallback_seat;
2899                *out = fallback_out;
2900            }
2901        }
2902        Ok(())
2903    }
2904
2905    /// Ask an implement seat's own CLI to confirm what it started, once, when
2906    /// its reply reported a command whose completion status it never
2907    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2908    /// that does and does not mean.
2909    ///
2910    /// The completion contract this task asks for, extended to `implement`
2911    /// with the same signal `continue_fix_report` reads for the fixer,
2912    /// rather than a keyword search over the reply or a hard requirement on
2913    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2914    /// a candidate's CLI turn ended cleanly while a test run it had started
2915    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2916    /// named in it at all is untouched by this: `commands` is empty, so
2917    /// there is nothing to be unconfirmed.
2918    ///
2919    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2920    /// this is not about recovering edits that might already be on disk, it
2921    /// is about a result the seat itself never vouched for, which resuming
2922    /// asks for regardless of what the tree already holds. Bounded to one
2923    /// attempt for the same reason `resume_undelivered` is — this is the
2924    /// most expensive node in the graph — and a seat that still cannot
2925    /// confirm on that attempt is left as whatever its (possibly still
2926    /// unconfirmed) reply says; this does not invent a new "failed" reason
2927    /// for a candidate that otherwise produced a real, committed change.
2928    async fn resume_unconfirmed_commands(
2929        &mut self,
2930        results: &mut [(usize, SeatState, AgentOutcome)],
2931        sent: &[SeatJob],
2932        prompts: &Prompts,
2933        run_id: &str,
2934    ) {
2935        for (wi, seat, out) in results.iter_mut() {
2936            let AgentOutcome::Ok(o) = &*out else {
2937                continue;
2938            };
2939            if !has_unconfirmed_command(&o.commands) {
2940                continue;
2941            }
2942            let Some(job) = sent.get(*wi) else { continue };
2943            if !has_context(&job.spec, seat, job.sessions) {
2944                self.state.event(
2945                    "implement",
2946                    format!(
2947                        "{}: the reply named a command whose own CLI never confirmed the exit \
2948                         status of, but there is no session left to resume",
2949                        seat.key
2950                    ),
2951                );
2952                continue;
2953            }
2954            self.state.event(
2955                "implement",
2956                format!(
2957                    "{}: the reply named a command whose own CLI never confirmed the exit \
2958                     status of; resuming the conversation",
2959                    seat.key
2960                ),
2961            );
2962            let mut retry = job.clone();
2963            retry.seat = seat.clone();
2964            retry.prompt = prompt::resume_incomplete(
2965                "a command in your last reply had no confirmed exit status",
2966            );
2967            retry.timeout = retry_budget(job.timeout, true);
2968            retry.stem = format!("{}-confirm", job.stem);
2969            let cache = self.state.config.cache_dir();
2970            let ctx = WaveCtx {
2971                carry_seats: false,
2972                pauses: Some((&self.pause, &self.interrupt)),
2973                run: run_id,
2974                node: "implement",
2975                prompts,
2976                cache: cache.as_deref(),
2977                round: None,
2978            };
2979            let (resumed_seat, resumed) =
2980                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2981            *seat = resumed_seat;
2982            *out = resumed;
2983        }
2984    }
2985
2986    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
2987    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
2988    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
2989    /// that motivated this.
2990    ///
2991    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
2992    /// own nudge loop already covers for judge/review/vote seats, and not a
2993    /// dropped stream, which [`Runner::resume_undelivered`] covers for
2994    /// implement seats: here the CLI turn genuinely finished while the node's
2995    /// own work — the fixer's account of what it did — had not. Gated purely
2996    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
2997    /// reply, never on any wording in it, so a fixer whose valid, first-try
2998    /// `FixReport` happens to mention having waited on a background test is
2999    /// never resumed — the `Ok(report)` branch at the call site returns
3000    /// before this is ever invoked.
3001    ///
3002    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3003    /// attempt ([`retry_budget`]), nothing attempted once the session is
3004    /// gone, and a quota hit ends the loop immediately rather than retrying a
3005    /// rate limit that fails the same way again.
3006    async fn continue_fix_report(
3007        &mut self,
3008        mut seat: SeatState,
3009        parse_err: String,
3010        job: &SeatJob,
3011        prompts: &Prompts,
3012        run_id: &str,
3013        round: usize,
3014    ) -> (
3015        SeatState,
3016        Option<FixReport>,
3017        Option<String>,
3018        ContinuationRecord,
3019    ) {
3020        let mut last_err = parse_err;
3021        let mut cumulative_wait_ms = 0u64;
3022        let mut attempts = 0usize;
3023        loop {
3024            if !has_context(&job.spec, &seat, job.sessions) {
3025                self.state.event(
3026                    "fix",
3027                    format!(
3028                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3029                         session left to resume into"
3030                    ),
3031                );
3032                let outcome = if attempts == 0 {
3033                    ContinuationOutcome::NoSession
3034                } else {
3035                    ContinuationOutcome::Exhausted
3036                };
3037                return (
3038                    seat,
3039                    None,
3040                    Some(format!("unparsable fix report: {last_err}")),
3041                    ContinuationRecord {
3042                        attempts,
3043                        cumulative_wait_ms,
3044                        outcome,
3045                    },
3046                );
3047            }
3048            if attempts >= MAX_FIX_CONTINUATIONS {
3049                self.state.event(
3050                    "fix",
3051                    format!(
3052                        "round {round}: fixer's reply still had no adoption report after \
3053                         {attempts} continuation(s) ({last_err}); giving up"
3054                    ),
3055                );
3056                return (
3057                    seat,
3058                    None,
3059                    Some(format!(
3060                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3061                    )),
3062                    ContinuationRecord {
3063                        attempts,
3064                        cumulative_wait_ms,
3065                        outcome: ContinuationOutcome::Exhausted,
3066                    },
3067                );
3068            }
3069            attempts += 1;
3070            self.state.event(
3071                "fix",
3072                format!(
3073                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3074                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3075                ),
3076            );
3077            let mut retry = job.clone();
3078            retry.seat = seat.clone();
3079            retry.prompt = prompt::resume_incomplete(&last_err);
3080            retry.timeout = retry_budget(job.timeout, true);
3081            retry.stem = format!("{}-continue{attempts}", job.stem);
3082            let cache = self.state.config.cache_dir();
3083            let ctx = WaveCtx {
3084                carry_seats: false,
3085                pauses: Some((&self.pause, &self.interrupt)),
3086                run: run_id,
3087                node: "fix",
3088                prompts,
3089                cache: cache.as_deref(),
3090                round: Some(round),
3091            };
3092            let (resumed_seat, resumed_out) = run_one(
3093                retry,
3094                Arc::clone(&self.sem),
3095                &ctx,
3096                &mut self.state,
3097                attempts,
3098            )
3099            .await;
3100            seat = resumed_seat;
3101            match resumed_out {
3102                AgentOutcome::Ok(o) => {
3103                    cumulative_wait_ms += o.duration_ms;
3104                    match verdict::extract_json::<FixReport>(&o.text) {
3105                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3106                            self.state.event(
3107                                "fix",
3108                                format!(
3109                                    "round {round}: fixer's adoption report recovered after \
3110                                     {attempts} continuation(s)"
3111                                ),
3112                            );
3113                            return (
3114                                seat,
3115                                Some(report),
3116                                None,
3117                                ContinuationRecord {
3118                                    attempts,
3119                                    cumulative_wait_ms,
3120                                    outcome: ContinuationOutcome::Resumed,
3121                                },
3122                            );
3123                        }
3124                        // The report parsed, but this same reply's own
3125                        // CommandEvidence — the identical record `state.jobs`
3126                        // renders — names a command whose CLI never
3127                        // confirmed an exit status. Read together, that is
3128                        // not a resolved answer: keep nudging rather than
3129                        // accept a report standing next to a command the
3130                        // seat's own CLI cannot vouch for.
3131                        Ok(_) => {
3132                            last_err = "the reply parsed, but it reported a command whose own CLI \
3133                                 never confirmed an exit status"
3134                                .to_owned();
3135                        }
3136                        Err(e) => last_err = e.to_string(),
3137                    }
3138                }
3139                AgentOutcome::Quota(o) => {
3140                    cumulative_wait_ms += o.duration_ms;
3141                    self.state.quota.push(QuotaLoss {
3142                        seat: seat.key.clone(),
3143                        node: "fix".to_owned(),
3144                        at: Timestamp::now(),
3145                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3146                    });
3147                    self.state.event(
3148                        "fix",
3149                        format!(
3150                            "round {round}: continuation rate limited (quota); not retrying now"
3151                        ),
3152                    );
3153                    return (
3154                        seat,
3155                        None,
3156                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3157                        ContinuationRecord {
3158                            attempts,
3159                            cumulative_wait_ms,
3160                            outcome: ContinuationOutcome::QuotaLost,
3161                        },
3162                    );
3163                }
3164                AgentOutcome::Dropped(o) => {
3165                    cumulative_wait_ms += o.duration_ms;
3166                    let why = o
3167                        .dropped
3168                        .as_ref()
3169                        .map(|d| d.why.as_str())
3170                        .unwrap_or("the CLI ended the stream without delivering its answer");
3171                    last_err = format!("the CLI dropped the stream ({why})");
3172                }
3173                AgentOutcome::Failed(e) => last_err = e,
3174                // The continuation was never started; the report stays lost
3175                // and the caller records that, without any handover.
3176                AgentOutcome::ResourceBlocked(e) => {
3177                    return (
3178                        seat,
3179                        None,
3180                        Some(format!("the continuation was not started: {e}")),
3181                        ContinuationRecord {
3182                            attempts: attempts.saturating_sub(1),
3183                            cumulative_wait_ms,
3184                            outcome: ContinuationOutcome::NoSession,
3185                        },
3186                    );
3187                }
3188            }
3189        }
3190    }
3191
3192    fn after_implement(&mut self) -> Result<()> {
3193        if !self.state.candidate_preservation.is_empty() {
3194            bail!("candidate preservation incomplete; judging is unsafe");
3195        }
3196        // Scan every candidate patch once the set is complete.
3197        if self.state.leaks.is_empty() {
3198            let cfg = self.state.config.blind.clone();
3199            let mut leaks = Vec::new();
3200            for c in &self.state.candidates {
3201                let Some(patch) =
3202                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3203                else {
3204                    continue;
3205                };
3206                leaks.extend(blind::scan(
3207                    &format!("candidate {} patch", c.label),
3208                    &patch,
3209                    &cfg.vendor_tokens,
3210                ));
3211            }
3212            if !leaks.is_empty() {
3213                let summary = leaks
3214                    .iter()
3215                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3216                    .collect::<Vec<_>>()
3217                    .join(", ");
3218                match cfg.on_leak {
3219                    LeakPolicy::Fail => {
3220                        self.state.status = RunStatus::Failed;
3221                        self.state
3222                            .event("blind", format!("vendor text in a patch: {summary}"));
3223                        self.state.leaks = leaks;
3224                        self.state.save()?;
3225                        self.settle_questions();
3226                        bail!(
3227                            "blind.on_leak = \"fail\" and vendor text reached a \
3228                             judged patch: {summary}"
3229                        );
3230                    }
3231                    LeakPolicy::Redact => self.state.event(
3232                        "blind",
3233                        format!("redacting vendor text for judging: {summary}"),
3234                    ),
3235                    LeakPolicy::Warn => self.state.event(
3236                        "blind",
3237                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3238                    ),
3239                }
3240                self.state.leaks = leaks;
3241            }
3242        }
3243
3244        if self.state.viable().is_empty() {
3245            if self.state.all_candidates_verified_noop() {
3246                // Every candidate agreed, with evidence the adoption guard
3247                // accepted, that nothing belongs in this worktree. That is
3248                // not the same fact as a candidate that simply failed to
3249                // write anything, and settling it as an ordinary `Failed`
3250                // (see `SCHEMA`'s doc for schema 10) is what let two of
3251                // task 391f's attempts burn a retry each re-discovering the
3252                // same already-landed fix. Terminal either way, so `judge`
3253                // must never run over an empty candidate set — unlike the
3254                // `Failed` branch below this returns `Ok`, not an error:
3255                // nothing here failed.
3256                self.state.status = RunStatus::VerifiedNoop;
3257                self.state.save()?;
3258                self.settle_questions();
3259                return Ok(());
3260            }
3261            self.state.status = RunStatus::Failed;
3262            self.state.save()?;
3263            self.settle_questions();
3264            bail!("no candidate produced a change; nothing to judge");
3265        }
3266        self.state.status = RunStatus::Judging;
3267        self.state.save()?;
3268        Ok(())
3269    }
3270
3271    // --------------------------------------------------------------- judge
3272
3273    async fn judge(&mut self) -> Result<()> {
3274        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3275        // agent files with `magi task add` name the run that paid for it. The
3276        // prompt overlay is cloned alongside it because the waves borrow it
3277        // while `self` is mutably borrowed by the node's own bookkeeping.
3278        let run_id = self.state.id.clone();
3279        let prompts = self.state.config.prompts.clone();
3280        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3281            return Ok(());
3282        }
3283        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3284        if viable.len() == 1 {
3285            // Recorded so this is a one-time event: `judgements` stays empty
3286            // either way, which without this flag is indistinguishable from
3287            // "not yet judged" on the next reentry — and status is left
3288            // untouched, so a later node's conclusion (e.g. `Blocked` after
3289            // the review budget ran out) survives a resume instead of being
3290            // clobbered back to `Judging` by this node running again.
3291            self.state.judge_skipped = true;
3292            self.state.event(
3293                "judge",
3294                format!(
3295                    "only candidate {} produced a change; judging skipped",
3296                    viable[0].label
3297                ),
3298            );
3299            self.state.save()?;
3300            return Ok(());
3301        }
3302        self.state.status = RunStatus::Judging;
3303
3304        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3305        let language = self.state.config.graph.language.clone();
3306        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3307        let sessions = self.state.config.graph.sessions;
3308        let artifacts = agent::artifacts_dir(&self.state.dir());
3309        let root = self.state.worktree_root();
3310        let base_short = short(&self.state.base_commit);
3311
3312        let mut jobs = Vec::new();
3313        let mut orders = Vec::new();
3314        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3315            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3316            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3317            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3318            let seat_key = format!("judge-{}", j + 1);
3319            let seat = self.seat(&seat_key, &spec.id);
3320            jobs.push(SeatJob {
3321                prompt: prompt::judge(
3322                    &self.state.instruction,
3323                    &views,
3324                    self.roles.judges.len(),
3325                    &base_short,
3326                    &language,
3327                ),
3328                spec,
3329                seat,
3330                cwd: root.join(format!("judge-{}", j + 1)),
3331                timeout,
3332                allow_write: false,
3333                sessions,
3334                artifacts: artifacts.clone(),
3335                stem: format!("judge-{}", j + 1),
3336                handover: None,
3337            });
3338        }
3339
3340        self.state.event(
3341            "judge",
3342            format!(
3343                "{} judges ranking {} candidates blind",
3344                jobs.len(),
3345                viable.len()
3346            ),
3347        );
3348        let labels_for_check = labels.clone();
3349        let mut quota_losses = Vec::new();
3350        let cache = self.state.config.cache_dir();
3351        let ctx = WaveCtx {
3352            carry_seats: false,
3353            pauses: Some((&self.pause, &self.interrupt)),
3354            run: &run_id,
3355            node: "judge",
3356            prompts: &prompts,
3357            cache: cache.as_deref(),
3358            round: None,
3359        };
3360        let results = ask_json_wave::<Ranking>(
3361            jobs,
3362            Arc::clone(&self.sem),
3363            self.state.config.graph.retries,
3364            &self.roles.judge_roster,
3365            &ctx,
3366            &mut quota_losses,
3367            &mut self.state,
3368            &move |r: &Ranking| r.validate(&labels_for_check),
3369        )
3370        .await;
3371        if self.state.resource_block.is_some() {
3372            return self.stop_for_resource();
3373        }
3374        self.state.quota.extend(quota_losses);
3375
3376        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3377            let agent_id = seat.agent.clone();
3378            self.state.seats.insert(seat.key.clone(), seat);
3379            let mut record = Judgement {
3380                judge: j + 1,
3381                seat: format!("judge-{}", j + 1),
3382                agent: agent_id,
3383                ranking: Vec::new(),
3384                reasons: BTreeMap::new(),
3385                confidence: None,
3386                strengths: BTreeMap::new(),
3387                order: orders[j].clone(),
3388                failed: None,
3389                duration_ms: 0,
3390            };
3391            match res {
3392                Ok((ranking, out)) => {
3393                    record.ranking = ranking.normalized();
3394                    record.reasons = ranking.reasons;
3395                    record.strengths = ranking.strengths;
3396                    record.confidence = ranking.confidence;
3397                    record.duration_ms = out.duration_ms;
3398                    self.state.event(
3399                        "judge",
3400                        format!(
3401                            "judge {} ranked {}",
3402                            j + 1,
3403                            record.ranking.iter().collect::<String>()
3404                        ),
3405                    );
3406                }
3407                Err(e) => {
3408                    record.failed = Some(e.to_string());
3409                    self.state
3410                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3411                }
3412            }
3413            self.state.judgements.push(record);
3414            self.state.save()?;
3415        }
3416        Ok(())
3417    }
3418
3419    // ---------------------------------------------------------- deliberate
3420
3421    async fn deliberate(&mut self) -> Result<()> {
3422        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3423        // agent files with `magi task add` name the run that paid for it. The
3424        // prompt overlay is cloned alongside it because the waves borrow it
3425        // while `self` is mutably borrowed by the node's own bookkeeping.
3426        let run_id = self.state.id.clone();
3427        let prompts = self.state.config.prompts.clone();
3428        if !self.state.deliberation.is_empty() {
3429            return Ok(());
3430        }
3431        let tops: Vec<char> = self
3432            .state
3433            .judgements
3434            .iter()
3435            .filter_map(|j| j.ranking.first().copied())
3436            .collect();
3437        let rounds = self.state.config.graph.deliberate_rounds;
3438        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3439            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3440                self.state.event(
3441                    "deliberate",
3442                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3443                );
3444            }
3445            self.state.status = RunStatus::Voting;
3446            self.state.save()?;
3447            return Ok(());
3448        }
3449
3450        self.state.status = RunStatus::Deliberating;
3451        self.state.event(
3452            "deliberate",
3453            format!(
3454                "split: first choices were {} — opening {rounds} round(s)",
3455                tops.iter().collect::<String>()
3456            ),
3457        );
3458
3459        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3460        let language = self.state.config.graph.language.clone();
3461        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3462        let sessions = self.state.config.graph.sessions;
3463        let artifacts = agent::artifacts_dir(&self.state.dir());
3464        let root = self.state.worktree_root();
3465        let base_short = short(&self.state.base_commit);
3466
3467        // Judges argue in sequence so that a turn can answer the one before it;
3468        // that is the difference between deliberation and three parallel
3469        // monologues.
3470        for round in 1..=rounds {
3471            let mut turns: Vec<DeliberationTurn> = Vec::new();
3472            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3473                if self.state.judgements[j].failed.is_some() {
3474                    continue;
3475                }
3476                let seat_key = format!("judge-{}", j + 1);
3477                let spec = self.occupant(&seat_key, spec);
3478                let mut seat = self.seat(&seat_key, &spec.id);
3479                let transcript = self.transcript(&turns, j);
3480                let build = |context: Option<&str>| {
3481                    prompt::deliberate(
3482                        &self.state.instruction,
3483                        context,
3484                        &transcript,
3485                        round,
3486                        rounds,
3487                        &language,
3488                    )
3489                };
3490                let block = self.candidate_block(&viable, &base_short);
3491                let full = build(Some(&block));
3492                let text = if has_context(&spec, &seat, sessions) {
3493                    build(None)
3494                } else {
3495                    full.clone()
3496                };
3497                let job = SeatJob {
3498                    spec,
3499                    seat: seat.clone(),
3500                    prompt: text,
3501                    cwd: root.join(format!("judge-{}", j + 1)),
3502                    timeout,
3503                    allow_write: false,
3504                    sessions,
3505                    artifacts: artifacts.clone(),
3506                    stem: format!("delib-{round}-judge-{}", j + 1),
3507                    handover: Some(full),
3508                };
3509                let cache = self.state.config.cache_dir();
3510                let ctx = WaveCtx {
3511                    carry_seats: false,
3512                    pauses: Some((&self.pause, &self.interrupt)),
3513                    run: &run_id,
3514                    node: "deliberate",
3515                    prompts: &prompts,
3516                    cache: cache.as_deref(),
3517                    round: None,
3518                };
3519                // A turn is never nudged (`retries` 0): a failed seat is
3520                // handed to the next roster agent, which gets the full
3521                // context. An empty answer is a turn, not a failure.
3522                let mut losses = Vec::new();
3523                let mut results = ask_wave_with::<String>(
3524                    vec![job],
3525                    Arc::clone(&self.sem),
3526                    0,
3527                    &self.roles.judge_roster,
3528                    &ctx,
3529                    &mut losses,
3530                    &mut self.state,
3531                    &|text: &str| {
3532                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3533                    },
3534                )
3535                .await;
3536                self.state.quota.extend(losses);
3537                let (updated, res, _) = results.pop().expect("one job in, one result out");
3538                seat = updated;
3539                let agent_id = seat.agent.clone();
3540                self.state.seats.insert(seat.key.clone(), seat);
3541                let body = match res {
3542                    Ok((body, _)) => body,
3543                    // Skip the seat; a CLI's raw error JSON is never read as
3544                    // this judge's position.
3545                    Err(e) => {
3546                        self.state
3547                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3548                        continue;
3549                    }
3550                };
3551                let tentative = verdict::extract_json::<Position>(&body)
3552                    .ok()
3553                    .and_then(|p| p.tentative)
3554                    .and_then(|s| s.trim().chars().next())
3555                    .map(|c| c.to_ascii_uppercase());
3556                self.state.event(
3557                    "deliberate",
3558                    format!(
3559                        "round {round}: judge {} now favours {}",
3560                        j + 1,
3561                        tentative.map_or("—".to_owned(), |c| c.to_string())
3562                    ),
3563                );
3564                turns.push(DeliberationTurn {
3565                    judge: j + 1,
3566                    agent: agent_id,
3567                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3568                    tentative,
3569                });
3570            }
3571            self.state
3572                .deliberation
3573                .push(DeliberationRound { round, turns });
3574            self.state.save()?;
3575        }
3576
3577        self.state.status = RunStatus::Voting;
3578        self.state.save()?;
3579        Ok(())
3580    }
3581
3582    // ---------------------------------------------------------------- vote
3583
3584    async fn vote(&mut self) -> Result<()> {
3585        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3586        // agent files with `magi task add` name the run that paid for it. The
3587        // prompt overlay is cloned alongside it because the waves borrow it
3588        // while `self` is mutably borrowed by the node's own bookkeeping.
3589        let run_id = self.state.id.clone();
3590        let prompts = self.state.config.prompts.clone();
3591        if !self.state.votes.is_empty() {
3592            return Ok(());
3593        }
3594        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3595        if viable.len() == 1 {
3596            return Ok(());
3597        }
3598        self.state.status = RunStatus::Voting;
3599
3600        let language = self.state.config.graph.language.clone();
3601        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3602        let sessions = self.state.config.graph.sessions;
3603        let artifacts = agent::artifacts_dir(&self.state.dir());
3604        let root = self.state.worktree_root();
3605        let base_short = short(&self.state.base_commit);
3606        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3607
3608        let mut jobs = Vec::new();
3609        let mut seats_at = Vec::new();
3610        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3611            if self
3612                .state
3613                .judgements
3614                .get(j)
3615                .is_some_and(|r| r.failed.is_some())
3616            {
3617                continue;
3618            }
3619            let seat_key = format!("judge-{}", j + 1);
3620            let spec = self.occupant(&seat_key, spec);
3621            let seat = self.seat(&seat_key, &spec.id);
3622            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3623            let text = if has_context(&spec, &seat, sessions) {
3624                prompt::final_vote(&viable, &language)
3625            } else {
3626                full.clone()
3627            };
3628            jobs.push(SeatJob {
3629                spec,
3630                seat,
3631                prompt: text,
3632                cwd: root.join(format!("judge-{}", j + 1)),
3633                timeout,
3634                allow_write: false,
3635                sessions,
3636                artifacts: artifacts.clone(),
3637                stem: format!("vote-judge-{}", j + 1),
3638                handover: Some(full),
3639            });
3640            seats_at.push(j);
3641        }
3642
3643        self.state.event(
3644            "vote",
3645            format!(
3646                "collecting {} final votes one by one, privately",
3647                jobs.len()
3648            ),
3649        );
3650        let allowed = viable.clone();
3651        let mut quota_losses = Vec::new();
3652        let cache = self.state.config.cache_dir();
3653        let ctx = WaveCtx {
3654            carry_seats: false,
3655            pauses: Some((&self.pause, &self.interrupt)),
3656            run: &run_id,
3657            node: "vote",
3658            prompts: &prompts,
3659            cache: cache.as_deref(),
3660            round: None,
3661        };
3662        let results = ask_json_wave::<FinalVote>(
3663            jobs,
3664            Arc::clone(&self.sem),
3665            self.state.config.graph.retries,
3666            &self.roles.judge_roster,
3667            &ctx,
3668            &mut quota_losses,
3669            &mut self.state,
3670            &move |v: &FinalVote| match v.label() {
3671                Some(c) if allowed.contains(&c) => Ok(()),
3672                other => bail!("vote {other:?} is not one of {allowed:?}"),
3673            },
3674        )
3675        .await;
3676        if self.state.resource_block.is_some() {
3677            return self.stop_for_resource();
3678        }
3679        self.state.quota.extend(quota_losses);
3680
3681        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3682            let agent_id = seat.agent.clone();
3683            self.state.seats.insert(seat.key.clone(), seat);
3684            let initial = self
3685                .state
3686                .judgements
3687                .get(j)
3688                .and_then(|r| r.ranking.first().copied());
3689            let mut record = VoteRecord {
3690                judge: j + 1,
3691                agent: agent_id,
3692                vote: None,
3693                reason: String::new(),
3694                changed: false,
3695            };
3696            match res {
3697                Ok((v, _)) => {
3698                    record.vote = v.label();
3699                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3700                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3701                    self.state.event(
3702                        "vote",
3703                        format!(
3704                            "judge {} voted {}{}",
3705                            j + 1,
3706                            record.vote.unwrap_or('?'),
3707                            if record.changed { " (changed)" } else { "" }
3708                        ),
3709                    );
3710                }
3711                Err(e) => {
3712                    self.state
3713                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3714                }
3715            }
3716            self.state.votes.push(record);
3717            self.state.save()?;
3718        }
3719        Ok(())
3720    }
3721
3722    // --------------------------------------------------------------- tally
3723
3724    fn tally(&mut self) -> Result<()> {
3725        if self.state.tally.is_some() {
3726            return Ok(());
3727        }
3728        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3729        let tops: Vec<char> = self
3730            .state
3731            .judgements
3732            .iter()
3733            .filter_map(|j| j.ranking.first().copied())
3734            .collect();
3735        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3736
3737        // A judge whose private vote failed still counted once, in the initial
3738        // ranking; using it beats discarding a whole seat.
3739        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3740        let mut cast: Vec<char> = Vec::new();
3741        for (i, j) in self.state.judgements.iter().enumerate() {
3742            let vote = self
3743                .state
3744                .votes
3745                .iter()
3746                .find(|v| v.judge == i + 1)
3747                .and_then(|v| v.vote)
3748                .or_else(|| j.ranking.first().copied());
3749            if let Some(v) = vote {
3750                *first_choice.entry(v).or_insert(0) += 1;
3751                cast.push(v);
3752            }
3753        }
3754
3755        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3756        for j in &self.state.judgements {
3757            let n = j.ranking.len();
3758            for (pos, label) in j.ranking.iter().enumerate() {
3759                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3760            }
3761        }
3762
3763        let best = first_choice.values().copied().max().unwrap_or(0);
3764        let mut leaders: Vec<char> = first_choice
3765            .iter()
3766            .filter(|(_, v)| **v == best)
3767            .map(|(k, _)| *k)
3768            .collect();
3769        let mut tie_break = None;
3770        if leaders.len() > 1 {
3771            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3772            let borda_leaders: Vec<char> = leaders
3773                .iter()
3774                .copied()
3775                .filter(|l| borda[l] == top_borda)
3776                .collect();
3777            tie_break = Some(if borda_leaders.len() == 1 {
3778                format!(
3779                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3780                    leaders.len()
3781                )
3782            } else {
3783                format!(
3784                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3785                    leaders.len()
3786                )
3787            });
3788            leaders = borda_leaders;
3789            leaders.sort_unstable();
3790        }
3791        let winner = *leaders
3792            .first()
3793            .or(viable.first())
3794            .context("no candidate to declare a winner from")?;
3795
3796        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3797        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3798        let deliberated = !self.state.deliberation.is_empty();
3799
3800        // Whose verdict is this? A rate-limited seat is absent even if it
3801        // ranked before the limit hit, so presence is measured against the
3802        // recorded losses, not just "did a ranking ever appear".
3803        let quota_seats: std::collections::BTreeSet<&str> =
3804            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3805        let mut present = 0usize;
3806        for (i, j) in self.state.judgements.iter().enumerate() {
3807            if quota_seats.contains(j.seat.as_str()) {
3808                continue;
3809            }
3810            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3811            let voted = self
3812                .state
3813                .votes
3814                .iter()
3815                .any(|v| v.judge == i + 1 && v.vote.is_some());
3816            if ranked || voted {
3817                present += 1;
3818            }
3819        }
3820        // Strict majority of the configured panel. A bare majority is real
3821        // signal we can act on, while a minority verdict must never stand in
3822        // for a healthy one. A one-candidate run needs no panel at all, and
3823        // `judges` stays `0` rather than the roster size a panel that never
3824        // sat would otherwise be credited with.
3825        let needs_quorum = viable.len() > 1;
3826        let judges_total = if needs_quorum {
3827            self.roles.judges.len()
3828        } else {
3829            0
3830        };
3831        let quorum = if needs_quorum {
3832            judges_total / 2 + 1
3833        } else {
3834            0
3835        };
3836        let met_quorum = !needs_quorum || present >= quorum;
3837        let uncontested = (!needs_quorum).then(|| {
3838            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3839        });
3840
3841        self.state.event(
3842            "tally",
3843            match &uncontested {
3844                Some(reason) => format!("winner {winner} — {reason}"),
3845                None => format!(
3846                    "winner {winner} — votes {} | initial {} | {} changed | \
3847                     {present}/{judges_total} judges{}",
3848                    first_choice
3849                        .iter()
3850                        .map(|(k, v)| format!("{k}:{v}"))
3851                        .collect::<Vec<_>>()
3852                        .join(" "),
3853                    if unanimous_initial {
3854                        "unanimous"
3855                    } else {
3856                        "split"
3857                    },
3858                    changed_votes,
3859                    if met_quorum {
3860                        String::new()
3861                    } else {
3862                        format!(" — below quorum ({quorum} required)")
3863                    },
3864                ),
3865            },
3866        );
3867        if !met_quorum {
3868            self.state.event(
3869                "stall",
3870                format!(
3871                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3872                     the run stops here, resumable"
3873                ),
3874            );
3875        }
3876        self.state.tally = Some(Tally {
3877            first_choice,
3878            borda,
3879            winner,
3880            rankings: tops.len(),
3881            unanimous_initial,
3882            deliberated,
3883            changed_votes,
3884            unanimous_final,
3885            tie_break,
3886            judges: judges_total,
3887            present,
3888            quorum,
3889            met_quorum,
3890            uncontested,
3891        });
3892        self.state.status = if met_quorum {
3893            RunStatus::Reviewing
3894        } else {
3895            RunStatus::Stalled
3896        };
3897        self.state.save()?;
3898        Ok(())
3899    }
3900
3901    // ------------------------------------------------------------- recover
3902
3903    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3904    /// resumed toward completion once the transient cause clears.
3905    ///
3906    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3907    /// it toward the quorum, which is exactly the set of seats whose absence
3908    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3909    /// not depend on which node happened to hit the limit), or an ordinary
3910    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3911    /// seat is never disturbed.
3912    ///
3913    /// A seat that now answers with a usable ranking is "recovered": its
3914    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3915    /// `tally` counts it present again), and its vote re-collected. A seat that
3916    /// still fails keeps its loss and stays absent.
3917    ///
3918    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3919    /// to review/gate/merge), `false` when it is still below quorum (the run
3920    /// stays `Stalled`, still resumable for a later retry).
3921    #[allow(clippy::too_many_lines)]
3922    async fn recover_stall(&mut self) -> Result<bool> {
3923        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3924        // agent files with `magi task add` name the run that paid for it. The
3925        // prompt overlay is cloned alongside it because the waves borrow it
3926        // while `self` is mutably borrowed by the node's own bookkeeping.
3927        let run_id = self.state.id.clone();
3928        let prompts = self.state.config.prompts.clone();
3929        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3930        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3931        // is what keeps a non-quota collapse — or a quota loss recorded at the
3932        // deliberate node — from being a permanent dead-end on `--resume`.
3933        let quota_seats: BTreeSet<&str> =
3934            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3935        let absent: Vec<String> = self
3936            .state
3937            .judgements
3938            .iter()
3939            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3940            .map(|j| j.seat.clone())
3941            .collect();
3942        if absent.is_empty() {
3943            return Ok(false);
3944        }
3945        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3946        if viable.len() <= 1 {
3947            return Ok(false);
3948        }
3949        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3950        let language = self.state.config.graph.language.clone();
3951        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3952        let sessions = self.state.config.graph.sessions;
3953        let artifacts = agent::artifacts_dir(&self.state.dir());
3954        let root = self.state.worktree_root();
3955        let base_short = short(&self.state.base_commit);
3956        let candidates: Vec<Candidate> = viable.clone();
3957
3958        // Map each absent seat key to its 0-based position in `roles.judges`.
3959        let mut positions: Vec<usize> = absent
3960            .iter()
3961            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3962            .collect();
3963        if positions.is_empty() {
3964            return Ok(false);
3965        }
3966        positions.sort_unstable();
3967        positions.dedup();
3968
3969        // Re-rank the lost seats, one blind prompt each.
3970        let mut judge_jobs = Vec::new();
3971        for &j in &positions {
3972            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3973            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3974            let seat_key = format!("judge-{}", j + 1);
3975            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3976            let seat = self.seat(&seat_key, &spec.id);
3977            judge_jobs.push(SeatJob {
3978                spec,
3979                seat,
3980                prompt: prompt::judge(
3981                    &self.state.instruction,
3982                    &views,
3983                    self.roles.judges.len(),
3984                    &base_short,
3985                    &language,
3986                ),
3987                cwd: root.join(seat_key),
3988                timeout,
3989                allow_write: false,
3990                sessions,
3991                artifacts: artifacts.clone(),
3992                stem: format!("judge-{}-recover", j + 1),
3993                handover: None,
3994            });
3995        }
3996
3997        let labels_for_check = labels.clone();
3998        let mut judge_losses = Vec::new();
3999        let retries = self.state.config.graph.retries;
4000        let cache = self.state.config.cache_dir();
4001        let ctx = WaveCtx {
4002            carry_seats: false,
4003            pauses: Some((&self.pause, &self.interrupt)),
4004            run: &run_id,
4005            node: "judge",
4006            prompts: &prompts,
4007            cache: cache.as_deref(),
4008            round: None,
4009        };
4010        let results = ask_json_wave::<Ranking>(
4011            judge_jobs,
4012            Arc::clone(&self.sem),
4013            retries,
4014            &self.roles.judge_roster,
4015            &ctx,
4016            &mut judge_losses,
4017            &mut self.state,
4018            &move |r: &Ranking| r.validate(&labels_for_check),
4019        )
4020        .await;
4021        if self.state.resource_block.is_some() {
4022            self.stop_for_resource()?;
4023            return Ok(false);
4024        }
4025
4026        // Refresh the judgement of every seat that ranked again.
4027        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4028        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4029            let agent_id = seat.agent.clone();
4030            self.state.seats.insert(seat.key.clone(), seat);
4031            let record = &mut self.state.judgements[j];
4032            match res {
4033                Ok((ranking, out)) => {
4034                    record.agent = agent_id;
4035                    record.ranking = ranking.normalized();
4036                    record.reasons = ranking.reasons;
4037                    record.strengths = ranking.strengths;
4038                    record.confidence = ranking.confidence;
4039                    record.failed = None;
4040                    record.duration_ms = out.duration_ms;
4041                    recovered.insert(j);
4042                    self.state.event(
4043                        "recover",
4044                        format!("judge {} ranked again after the limit", j + 1),
4045                    );
4046                }
4047                Err(e) => {
4048                    self.state
4049                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4050                }
4051            }
4052        }
4053
4054        // Re-ask the votes of the seats that recovered a ranking.
4055        let mut vote_jobs = Vec::new();
4056        let mut vote_pos: Vec<usize> = Vec::new();
4057        for &j in &recovered {
4058            let seat_key = format!("judge-{}", j + 1);
4059            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4060            let seat = self.seat(&seat_key, &spec.id);
4061            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4062            let text = if has_context(&spec, &seat, sessions) {
4063                prompt::final_vote(&labels, &language)
4064            } else {
4065                full.clone()
4066            };
4067            vote_jobs.push(SeatJob {
4068                spec,
4069                seat,
4070                prompt: text,
4071                cwd: root.join(seat_key),
4072                timeout,
4073                allow_write: false,
4074                sessions,
4075                artifacts: artifacts.clone(),
4076                stem: format!("vote-judge-{}-recover", j + 1),
4077                handover: Some(full),
4078            });
4079            vote_pos.push(j);
4080        }
4081        let allowed = labels.clone();
4082        let mut vote_losses = Vec::new();
4083        let vote_retries = self.state.config.graph.retries;
4084        let vote_cache = self.state.config.cache_dir();
4085        let ctx = WaveCtx {
4086            carry_seats: false,
4087            pauses: Some((&self.pause, &self.interrupt)),
4088            run: &run_id,
4089            node: "vote",
4090            prompts: &prompts,
4091            cache: vote_cache.as_deref(),
4092            round: None,
4093        };
4094        let votes = ask_json_wave::<FinalVote>(
4095            vote_jobs,
4096            Arc::clone(&self.sem),
4097            vote_retries,
4098            &self.roles.judge_roster,
4099            &ctx,
4100            &mut vote_losses,
4101            &mut self.state,
4102            &move |v: &FinalVote| match v.label() {
4103                Some(c) if allowed.contains(&c) => Ok(()),
4104                other => bail!("vote {other:?} is not one of {allowed:?}"),
4105            },
4106        )
4107        .await;
4108        if self.state.resource_block.is_some() {
4109            self.stop_for_resource()?;
4110            return Ok(false);
4111        }
4112        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4113            let agent_id = seat.agent.clone();
4114            self.state.seats.insert(seat.key.clone(), seat);
4115            match res {
4116                Ok((v, _)) => {
4117                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4118                        rec.vote = v.label();
4119                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4120                    } else {
4121                        self.state.votes.push(VoteRecord {
4122                            judge: j + 1,
4123                            agent: agent_id,
4124                            vote: v.label(),
4125                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4126                            changed: false,
4127                        });
4128                    }
4129                    self.state.event(
4130                        "recover",
4131                        format!("judge {} voted again after the limit", j + 1),
4132                    );
4133                }
4134                Err(e) => {
4135                    self.state
4136                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4137                }
4138            }
4139        }
4140
4141        // A seat that ranked again is present even if its re-vote failed —
4142        // `tally` falls back to the initial ranking's first choice — so clear
4143        // its quota loss. Seats that still fail keep theirs and stay absent.
4144        let recovered_keys: BTreeSet<String> = recovered
4145            .iter()
4146            .map(|&j| format!("judge-{}", j + 1))
4147            .collect();
4148        self.state
4149            .quota
4150            .retain(|q| !recovered_keys.contains(&q.seat));
4151        // A seat that hit the limit again is a fresh loss, not the old one:
4152        // replace the stale entry so the history stays one-per-seat and the
4153        // daemon can tell this attempt's loss from a previous session's.
4154        for loss in judge_losses.into_iter().chain(vote_losses) {
4155            if recovered_keys.contains(&loss.seat) {
4156                continue;
4157            }
4158            self.state.quota.retain(|q| q.seat != loss.seat);
4159            self.state.quota.push(loss);
4160        }
4161
4162        // Recompute the verdict from the refreshed panel.
4163        self.state.tally = None;
4164        self.tally()?;
4165        Ok(self
4166            .state
4167            .tally
4168            .as_ref()
4169            .map(|t| t.met_quorum)
4170            .unwrap_or(false))
4171    }
4172
4173    // ---------------------------------------------------------------- graft
4174
4175    /// Port the strengths the judges attributed to the losing candidates onto
4176    /// the winner, once, between the tally and the review loop.
4177    ///
4178    /// Best-effort by construction: it returns nothing and fails nothing. A
4179    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4180    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4181    /// the run carries on as if the phase were off. See [`graft`] for the
4182    /// decisions and `AGENTS.md` for the invariants.
4183    ///
4184    /// The one exception is a rollback that could not be confirmed: the
4185    /// rejected commit may still be the winner's tip, so the run is blocked
4186    /// (resumable; the record stays `Running`, which makes a resume retry the
4187    /// restore) instead of reviewing, gating and merging it.
4188    async fn graft(&mut self) -> Result<()> {
4189        let Err(e) = self.graft_inner().await else {
4190            return Ok(());
4191        };
4192        if e.is::<crate::repo_lock::ResourceBlocked>() {
4193            return Err(e);
4194        }
4195        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4196            self.state.status = RunStatus::Blocked;
4197            self.state.event("graft", format!("blocked: {e:#}"));
4198            self.state.save()?;
4199            return Err(e);
4200        }
4201        // An error after the `Running` record was saved may have left the
4202        // fixer's work in the tree: the record must not stay unsettled
4203        // (a later resume would reset past the review fixes), so confirm the
4204        // restore now, or block.
4205        if let Some(rec) = self.state.graft.clone()
4206            && !rec.outcome.settled()
4207        {
4208            if let Some(winner) = self.state.winner().cloned() {
4209                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4210                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4211                        return Err(re);
4212                    }
4213                    self.state.status = RunStatus::Blocked;
4214                    self.state.event("graft", format!("blocked: {re:#}"));
4215                    self.state.save()?;
4216                    return Err(re);
4217                }
4218            }
4219            if let Some(g) = self.state.graft.as_mut() {
4220                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4221            }
4222        }
4223        self.state.event("graft", format!("skipped: {e:#}"));
4224        if let Err(e) = self.state.save() {
4225            tracing::warn!("save after graft: {e:#}");
4226        }
4227        Ok(())
4228    }
4229
4230    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4231    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4232    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4233        let _coordination = self.coordinate(wt).await?;
4234        crate::recovery::restore_repair(wt, pre)
4235            .await
4236            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4237    }
4238
4239    /// A skip is a run event, recorded once however often the node is re-entered.
4240    fn graft_skipped(&mut self, why: &str) {
4241        let message = format!("skipped: {why}");
4242        if !self
4243            .state
4244            .events
4245            .iter()
4246            .any(|e| e.node == "graft" && e.message == message)
4247        {
4248            self.state.event("graft", message);
4249        }
4250    }
4251
4252    async fn graft_inner(&mut self) -> Result<()> {
4253        // A round that was started and never finished (crash, kill): the
4254        // fixer may have left anything in the tree, so put the winner back.
4255        if let Some(rec) = self.state.graft.clone()
4256            && !rec.outcome.settled()
4257        {
4258            // Reviews exist only if the graft ended without its record being
4259            // settled (a swallowed error): the tree has moved on since, and
4260            // a reset to `pre` would discard the review fixes.
4261            if self.state.reviews.is_empty()
4262                && let Some(winner) = self.state.winner().cloned()
4263            {
4264                self.graft_restore(&winner.worktree, &rec.pre).await?;
4265            }
4266            if rec.outcome == GraftOutcome::WaitingForRestore {
4267                self.state.graft = None;
4268                self.state.save()?;
4269            } else {
4270                if let Some(g) = self.state.graft.as_mut() {
4271                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4272                }
4273                self.state.event(
4274                    "graft",
4275                    "rolled back: the previous round was interrupted; the winner is untouched",
4276                );
4277                self.state.save()?;
4278                return Ok(());
4279            }
4280        }
4281        let plan = match graft::plan(&self.state) {
4282            Ok(plan) => plan,
4283            Err(skip) => {
4284                if skip != graft::Skip::AlreadyDone {
4285                    self.graft_skipped(skip.reason());
4286                    self.state.save()?;
4287                }
4288                return Ok(());
4289            }
4290        };
4291        let Some(winner) = self.state.winner().cloned() else {
4292            return Ok(());
4293        };
4294        let wt = winner.worktree.clone();
4295        if !git::is_clean(&wt).await? {
4296            self.graft_skipped("the winner's worktree has uncommitted changes");
4297            self.state.save()?;
4298            return Ok(());
4299        }
4300        let pre = git::rev_parse(&wt, "HEAD").await?;
4301        // Saved before the fixer is called: a stop from here on is repaired
4302        // by the branch above on the next entry.
4303        self.state.graft = Some(GraftRecord {
4304            pre: pre.clone(),
4305            post: None,
4306            lines: None,
4307            limit: None,
4308            agent: None,
4309            outcome: GraftOutcome::Running,
4310            requested: graft::requested(&plan),
4311            adoption: None,
4312            adoption_error: None,
4313        });
4314        self.state.event(
4315            "graft",
4316            format!(
4317                "porting strengths of candidate(s) {} onto {}",
4318                plan.strengths
4319                    .iter()
4320                    .map(|(l, _, _)| l.to_string())
4321                    .collect::<Vec<_>>()
4322                    .join(", "),
4323                plan.winner
4324            ),
4325        );
4326        self.state.save()?;
4327
4328        let fix_prompt = prompt::graft(
4329            &self.state.instruction,
4330            plan.winner,
4331            &plan.strengths,
4332            &self.state.config.graph.language,
4333        );
4334        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4335        let sessions = self.state.config.graph.sessions;
4336        let artifacts = agent::artifacts_dir(&self.state.dir());
4337        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4338        let (_, seat, out) = self
4339            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4340                SeatJob {
4341                    prompt: fix_prompt.clone(),
4342                    spec,
4343                    seat,
4344                    cwd: wt.clone(),
4345                    timeout,
4346                    allow_write: true,
4347                    sessions,
4348                    artifacts: artifacts.clone(),
4349                    stem: "graft".to_owned(),
4350                    handover: None,
4351                }
4352            })
4353            .await;
4354        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4355            // Preserve the need to restore earlier fallback edits without
4356            // confusing an unstarted seat with an interrupted invocation.
4357            self.state.graft.as_mut().expect("graft record").outcome =
4358                GraftOutcome::WaitingForRestore;
4359            return self.stop_for_resource();
4360        }
4361        let mut why: Option<String> = None;
4362        match &out {
4363            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4364                why = Some("the fixer produced nothing usable".to_owned());
4365            }
4366            AgentOutcome::Ok(_) => {}
4367            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4368            AgentOutcome::Quota(o) => {
4369                self.state.quota.push(QuotaLoss {
4370                    seat: seat.key.clone(),
4371                    node: "graft".to_owned(),
4372                    at: Timestamp::now(),
4373                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4374                });
4375                why = Some("rate limited (quota)".to_owned());
4376            }
4377            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4378                why = Some(format!(
4379                    "the fixer failed: {}",
4380                    e.lines()
4381                        .next()
4382                        .unwrap_or("")
4383                        .chars()
4384                        .take(160)
4385                        .collect::<String>()
4386                ));
4387            }
4388        }
4389        let agent_id = seat.agent.clone();
4390        self.state.seats.insert(seat.key.clone(), seat);
4391
4392        let mut post = None;
4393        let mut lines = None;
4394        let _coordination = self.coordinate(&wt).await?;
4395        if why.is_none() {
4396            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4397            if let Ok(r) = &rescued {
4398                self.state.note_withheld("graft", &r.withheld);
4399            }
4400            let head = git::rev_parse(&wt, "HEAD").await?;
4401            // What is measured must be what gets reviewed and gated: a failed
4402            // rescue, or anything still uncommitted (withheld lockfiles
4403            // aside, which are never part of the tree), rejects the graft.
4404            let dirty = match &rescued {
4405                Ok(r) => git::status_porcelain(&wt)
4406                    .await
4407                    .map(|st| {
4408                        st.lines().any(|l| {
4409                            !r.withheld
4410                                .iter()
4411                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4412                        })
4413                    })
4414                    .unwrap_or(true),
4415                Err(_) => true,
4416            };
4417            if dirty {
4418                why = Some("the fixer left changes that could not be committed".to_owned());
4419            } else if head == pre {
4420                why = Some("the fixer changed nothing".to_owned());
4421            } else {
4422                lines = git::changed_lines(&wt, &pre, &head).await;
4423                post = Some(head);
4424            }
4425        }
4426        let (adoption, adoption_error) = match &out {
4427            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4428            _ => (None, None),
4429        };
4430        let requested = graft::requested(&plan);
4431        let outcome = match why {
4432            None => {
4433                self.state.event(
4434                    "graft",
4435                    format!(
4436                        "applied: {}",
4437                        lines.map_or_else(
4438                            || "changed lines unavailable".to_owned(),
4439                            |n| format!("{n} changed lines")
4440                        )
4441                    ),
4442                );
4443                GraftOutcome::Applied
4444            }
4445            Some(why) => {
4446                // Not settled until the restore is confirmed: the record
4447                // stays `Running`, so a resume retries it.
4448                crate::recovery::restore_repair(&wt, &pre)
4449                    .await
4450                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4451                self.state.event(
4452                    "graft",
4453                    format!("rolled back: {why}; the winner is untouched"),
4454                );
4455                GraftOutcome::RolledBack(why)
4456            }
4457        };
4458        self.state.graft = Some(GraftRecord {
4459            pre,
4460            post,
4461            lines,
4462            limit: None,
4463            agent: Some(agent_id),
4464            outcome,
4465            requested,
4466            adoption,
4467            adoption_error,
4468        });
4469        self.state.save()?;
4470        Ok(())
4471    }
4472
4473    // ----------------------------------------------------------------- fold
4474
4475    async fn fold_losers(&mut self) -> Result<()> {
4476        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4477            return Ok(());
4478        };
4479        if !self.state.candidate_preservation.is_empty() {
4480            bail!("candidate preservation incomplete; refusing cleanup");
4481        }
4482        let repo = self.state.repo.clone();
4483        let _coordination = self.coordinate(&repo).await?;
4484        let mut folded = Vec::new();
4485        for i in 0..self.state.candidates.len() {
4486            let c = &self.state.candidates[i];
4487            if c.label == winner || c.folded {
4488                continue;
4489            }
4490            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4491            remove_preserved_worktree(&repo, &wt).await?;
4492            if git::branch_exists(&repo, &branch).await?
4493                && !git::branch_delete(&repo, &branch).await?
4494            {
4495                bail!("candidate branch removal refused");
4496            }
4497            self.state.candidates[i].folded = true;
4498            self.state.save()?;
4499            folded.push(label.to_string());
4500        }
4501        // The judges are finished; their checkouts are pure cost from here.
4502        let root = self.state.worktree_root();
4503        for j in 1..=self.roles.judges.len() {
4504            let wt = root.join(format!("judge-{j}"));
4505            if wt.exists() {
4506                remove_preserved_worktree(&repo, &wt).await?;
4507            }
4508        }
4509        // The design-deliberation stage is finished by the time a tally
4510        // exists — same reasoning as the judges above.
4511        if self.state.config.graph.advise {
4512            for k in 1..=self.state.config.graph.advisors {
4513                let wt = root.join(format!("advisor-{k}"));
4514                if wt.exists() {
4515                    remove_preserved_worktree(&repo, &wt).await?;
4516                }
4517            }
4518        }
4519        if !folded.is_empty() {
4520            self.state
4521                .event("fold", format!("folded candidates {}", folded.join(", ")));
4522            self.state.save()?;
4523        }
4524        Ok(())
4525    }
4526
4527    // ------------------------------------------------------------ base sync
4528
4529    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4530    /// anything verifies it.
4531    ///
4532    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4533    /// read whatever is checked out in the winner's worktree. Left alone that
4534    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4535    /// it when the run *branched* - and a run takes long enough that the base
4536    /// has usually moved by the time it gets here. A gate that ran there
4537    /// answers "green on the commit this run started from", not "green on
4538    /// what is about to land", and the difference showed up three times in
4539    /// one day as a green run whose merge would have reverted a file another
4540    /// pull request had already landed.
4541    ///
4542    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4543    /// `land::Step::Rebase` calls, rather than a second implementation of the
4544    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4545    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4546    /// doc warns about ("two answers to a question nobody notices until a
4547    /// diff is wrong").
4548    ///
4549    /// A conflict is not the end of the road: the standing rebase is handed
4550    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4551    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4552    /// land). Once it finishes, review and the gate run as usual on the
4553    /// rebased tree, which is where a breakage the new base caused is caught
4554    /// by the ordinary gate-fix round. magi resolves nothing itself.
4555    ///
4556    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4557    /// `state.base_sync.attempts`, is how many times the base is *rebased
4558    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4559    /// *conflict* was given to a fixer. When the fixer cannot finish the
4560    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4561    /// what was tried (rounds spent, paths still conflicted) and the branch
4562    /// and worktree stay exactly as they were - untouched, for a person to
4563    /// look at - which is also what makes re-entering this function
4564    /// afterwards a no-op instead of a second attempt at the same wall. A
4565    /// push failure ends the same way.
4566    async fn sync_to_base(&mut self) -> Result<()> {
4567        if self.state.status == RunStatus::AlreadyInBase {
4568            return Ok(());
4569        }
4570        let conflicted = self
4571            .state
4572            .base_sync
4573            .as_ref()
4574            .is_some_and(|s| s.conflict.is_some());
4575        let Some(winner) = self.state.winner().cloned() else {
4576            return Ok(());
4577        };
4578
4579        let repo = self.state.repo.clone();
4580        let remote = self.state.config.merge.remote.clone();
4581        let base_branch = self.state.base_branch.clone();
4582        let tracking = format!("{remote}/{base_branch}");
4583
4584        git::fetch(&repo, &remote, &base_branch).await.ok();
4585        // No network, or the remote never had this branch: the run already
4586        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4587        // otherwise), and one that got this far is not blocked by a fetch
4588        // that fails now. It just has no newer tip to compare against.
4589        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4590            return Ok(());
4591        };
4592
4593        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4594        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4595        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4596
4597        // Before any rebase, and before a recorded conflict is honoured: a
4598        // branch whose change reached the base under other commit ids has
4599        // nothing to rebase and nothing to conflict with, and a run that
4600        // already stopped on that phantom conflict recovers here on resume.
4601        // `behind == 0` with head == tip is a branch the base has since taken
4602        // in whole, whether or not a conflict was ever recorded: the ancestry
4603        // proof must still run (`classify` ignores a head still on the start
4604        // commit).
4605        if (behind > 0 || conflicted || head == tip)
4606            && self
4607                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4608                .await?
4609        {
4610            return Ok(());
4611        }
4612        if conflicted {
4613            return Ok(());
4614        }
4615
4616        if behind == 0 {
4617            // A fixer-finished rebase moves the branch ref before the
4618            // winner's worktree is told (`sync_to_head` below). A run that
4619            // died in between resumes here with `behind == 0` and a tree still
4620            // holding the pre-rebase files, which review and the gate would
4621            // then read. That state is exactly: HEAD moved off the tip the
4622            // rebase started from, yet the tree is still identical to that
4623            // tip. A tree with edits of its own differs from it, so nothing
4624            // is thrown away.
4625            if let Some(from) = self
4626                .state
4627                .rebase_fixes
4628                .iter()
4629                .rev()
4630                .find_map(|r| r.from.clone())
4631                && from != head
4632                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4633                    .await
4634                    .is_ok_and(|o| o.ok())
4635            {
4636                git::sync_to_head(&winner.worktree).await?;
4637            }
4638            // An earlier attempt may have rebased the branch locally and died
4639            // before pushing it (only the fresh-rebase arm below pushes).
4640            // Publish it now, so the plain push at PR time is not refused as
4641            // a non-fast-forward. A run already holding a recorded conflict
4642            // never reaches here; that case is out of scope.
4643            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4644            if let Some(why) = &conflict {
4645                self.state.status = RunStatus::Blocked;
4646                self.state.event("land", why.clone());
4647            }
4648            self.state.base_sync = Some(BaseSync {
4649                tip,
4650                behind: 0,
4651                attempts,
4652                conflict,
4653                already_in: None,
4654            });
4655            self.state.save()?;
4656            return Ok(());
4657        }
4658
4659        if attempts >= BASE_SYNC_ROUNDS {
4660            let why = format!(
4661                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4662                 rebase(s); rebasing again would only race it",
4663                winner.branch
4664            );
4665            self.state.status = RunStatus::Blocked;
4666            self.state.base_sync = Some(BaseSync {
4667                tip,
4668                behind,
4669                attempts,
4670                conflict: Some(why.clone()),
4671                already_in: None,
4672            });
4673            self.state.event("land", why);
4674            self.state.save()?;
4675            return Ok(());
4676        }
4677
4678        self.state.event(
4679            "land",
4680            format!(
4681                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4682                winner.branch
4683            ),
4684        );
4685        self.state.save()?;
4686
4687        // The remote's copy of the branch, read now and only if the fetch
4688        // really succeeded (a stale tracking ref must never pin a lease). It is
4689        // pushed over after a rebase only when it is a commit this branch
4690        // already contains, by ancestry or by patch (an earlier rebase of ours
4691        // that never reached the remote): anything else is somebody else's work.
4692        let branch_tracking = format!("{remote}/{}", winner.branch);
4693        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4694        let remote_tip = if self.state.task_pr_branch().is_none()
4695            && matches!(&fetched_branch, Ok(o) if o.ok())
4696        {
4697            git::rev_parse(&repo, &branch_tracking).await.ok()
4698        } else {
4699            None
4700        };
4701        // A remote tip this branch does not contain is somebody else's work:
4702        // rebasing would leave a local tip that can never be pushed. Stop
4703        // before touching anything and say so.
4704        if let Some(theirs) = &remote_tip
4705            && !git::is_ancestor(&repo, theirs, &head).await
4706            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4707                .await
4708                .is_ok_and(|missing| missing.is_empty())
4709        {
4710            let why = format!(
4711                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4712                 them",
4713                short(theirs),
4714                winner.branch
4715            );
4716            self.state.status = RunStatus::Blocked;
4717            self.state.base_sync = Some(BaseSync {
4718                tip,
4719                behind,
4720                attempts,
4721                conflict: Some(why.clone()),
4722                already_in: None,
4723            });
4724            self.state.event("land", why);
4725            self.state.save()?;
4726            return Ok(());
4727        }
4728
4729        let scratch = self.state.dir().join("base-sync");
4730        let rebased = match crate::rebase::rebase_with_fixer(
4731            &mut self.state,
4732            &scratch,
4733            &winner.branch,
4734            &tracking,
4735        )
4736        .await
4737        {
4738            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4739            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4740            Err(e) => Err(e),
4741        };
4742        let attempts = attempts + 1;
4743        match rebased {
4744            Ok(None) => {
4745                // The branch ref moved, but a worktree that already had it
4746                // checked out (the winner's) was not told; sync its index and
4747                // files before anything reads them.
4748                git::sync_to_head(&winner.worktree).await?;
4749                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4750                let mut conflict = None;
4751                if let Some(pinned) = &remote_tip {
4752                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4753                    match pushed {
4754                        Ok(o) if o.ok() => self.state.event(
4755                            "land",
4756                            format!("pushed rebased {} to {remote}", winner.branch),
4757                        ),
4758                        Ok(o) => {
4759                            conflict = Some(format!(
4760                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4761                                winner.branch,
4762                                short(pinned),
4763                                o.stderr.chars().take(600).collect::<String>()
4764                            ));
4765                        }
4766                        Err(e) => {
4767                            conflict = Some(format!(
4768                                "rebased {} locally but could not push it: {e:#}",
4769                                winner.branch
4770                            ));
4771                        }
4772                    }
4773                }
4774                if let Some(why) = &conflict {
4775                    self.state.status = RunStatus::Blocked;
4776                    self.state.event("land", why.clone());
4777                }
4778                self.state.base_sync = Some(BaseSync {
4779                    tip: tip.clone(),
4780                    behind: 0,
4781                    attempts,
4782                    conflict,
4783                    already_in: None,
4784                });
4785                self.state
4786                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4787            }
4788            Ok(Some(conflict)) => {
4789                let why = format!(
4790                    "{} conflicts with {tracking} and did not rebase: {}",
4791                    winner.branch,
4792                    conflict.chars().take(600).collect::<String>()
4793                );
4794                self.state.status = RunStatus::Blocked;
4795                self.state.base_sync = Some(BaseSync {
4796                    tip,
4797                    behind,
4798                    attempts,
4799                    conflict: Some(why.clone()),
4800                    already_in: None,
4801                });
4802                self.state.event("land", why);
4803            }
4804            // A refused Git write or a busy repository: nothing is wrong with
4805            // the branch. The scratch rebase stays standing and the run
4806            // resumes into it (`base_sync` must not claim a conflict).
4807            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4808                self.state.save()?;
4809                return Ok(());
4810            }
4811            Err(e) => {
4812                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4813                self.state.status = RunStatus::Blocked;
4814                self.state.base_sync = Some(BaseSync {
4815                    tip,
4816                    behind,
4817                    attempts,
4818                    conflict: Some(why.clone()),
4819                    already_in: None,
4820                });
4821                self.state.event("land", why);
4822            }
4823        }
4824        self.state.save()?;
4825        Ok(())
4826    }
4827
4828    /// Push a branch an earlier attempt rebased locally but never published,
4829    /// pinned to the remote tip read right after a successful fetch. Returns
4830    /// the reason when the run must stop; `None` when there was nothing to do
4831    /// (no remote copy, the same tip, or a remote copy this branch already
4832    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4833    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4834        // Task competitions publish only after the PR text gate, to their
4835        // public head; never resurrect a candidate branch on the remote.
4836        if self.state.task_pr_branch().is_some() {
4837            return None;
4838        }
4839        let repo = self.state.repo.clone();
4840        let remote = self.state.config.merge.remote.clone();
4841        let fetched = git::fetch(&repo, &remote, branch).await;
4842        if !matches!(&fetched, Ok(o) if o.ok()) {
4843            return None;
4844        }
4845        let branch_tracking = format!("{remote}/{branch}");
4846        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4847        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4848            return None;
4849        }
4850        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4851            .await
4852            .is_ok_and(|missing| missing.is_empty())
4853        {
4854            return Some(format!(
4855                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4856                 them",
4857                short(&theirs)
4858            ));
4859        }
4860        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4861            Ok(o) if o.ok() => {
4862                self.state
4863                    .event("land", format!("pushed rebased {branch} to {remote}"));
4864                None
4865            }
4866            Ok(o) => Some(format!(
4867                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4868                 someone may have pushed): {}",
4869                short(&theirs),
4870                o.stderr.chars().take(600).collect::<String>()
4871            )),
4872            Err(e) => Some(format!(
4873                "{branch} is rebased locally but could not be pushed: {e:#}"
4874            )),
4875        }
4876    }
4877
4878    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4879    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4880    /// whether it did.
4881    ///
4882    /// Checked only when the base is ahead of the branch. A failing check is
4883    /// "not proven" - the ordinary rebase path then decides - never a reason to
4884    /// stop the run.
4885    ///
4886    /// The remote copy of the branch is held to the same standard as the local
4887    /// one: if it carries a tip this worktree does not, that tip must itself be
4888    /// proven in the base, or nothing is settled (a pull request would
4889    /// otherwise be closed over commits nobody checked). The pull request is
4890    /// closed *before* the terminal status is saved; if that fails for a
4891    /// reason other than a refusal (no network, a `gh` error) the run is left
4892    /// `Blocked` with the reason as its conflict, which a resume retries -
4893    /// the same recovery a phantom conflict gets.
4894    async fn settle_already_in(
4895        &mut self,
4896        branch: &str,
4897        tip: &str,
4898        head: &str,
4899        attempts: usize,
4900        behind: usize,
4901    ) -> Result<bool> {
4902        let repo = self.state.repo.clone();
4903        let remote = self.state.config.merge.remote.clone();
4904        let start = self.state.base_commit.clone();
4905        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4906            Ok(Some(e)) => e,
4907            Ok(None) => return Ok(false),
4908            Err(e) => {
4909                tracing::warn!("already-in-base check for {branch}: {e:#}");
4910                return Ok(false);
4911            }
4912        };
4913        let mut verified = vec![head.to_owned()];
4914        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4915        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4916        if matches!(&fetched, Ok(o) if o.ok())
4917            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4918            && theirs != head
4919        {
4920            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4921                Ok(Some(_)) => verified.push(theirs),
4922                _ => return Ok(false),
4923            }
4924        }
4925        let base_branch = self.state.base_branch.clone();
4926        let message = format!(
4927            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4928             land",
4929            evidence.names(),
4930            evidence.proof.as_str()
4931        );
4932        let closed =
4933            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4934                .await;
4935        match closed {
4936            Ok(Ok(url)) => self
4937                .state
4938                .event("land", format!("closed {url}: superseded on {base_branch}")),
4939            Ok(Err(why)) => self
4940                .state
4941                .event("land", format!("did not close a pull request: {why}")),
4942            Err(e) => {
4943                let why = format!(
4944                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4945                     not be closed ({e:#}); resume to retry"
4946                );
4947                self.state.status = RunStatus::Blocked;
4948                self.state.base_sync = Some(BaseSync {
4949                    tip: tip.to_owned(),
4950                    behind,
4951                    attempts,
4952                    conflict: Some(why.clone()),
4953                    already_in: None,
4954                });
4955                self.state.event("land", why);
4956                self.state.save()?;
4957                return Ok(true);
4958            }
4959        }
4960        self.state.status = RunStatus::AlreadyInBase;
4961        self.state.base_sync = Some(BaseSync {
4962            tip: tip.to_owned(),
4963            behind,
4964            attempts,
4965            conflict: None,
4966            already_in: Some(evidence),
4967        });
4968        self.state.event("land", message);
4969        self.state.save()?;
4970        self.settle_questions();
4971        Ok(true)
4972    }
4973
4974    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
4975    /// last landed the winner on, once it has run, else the commit the run
4976    /// branched from.
4977    ///
4978    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
4979    /// and `vote` all happen before there is a winner to rebase, so they
4980    /// compare every candidate against the branch point on purpose, and a
4981    /// base that moves after they are already done cannot change an answer
4982    /// they already gave.
4983    fn landing_base(&self) -> String {
4984        self.state
4985            .base_sync
4986            .as_ref()
4987            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4988    }
4989
4990    // ------------------------------------------------------- operator fix
4991
4992    /// Route specific, already-recorded review findings to a fixer for a
4993    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
4994    /// entry point.
4995    ///
4996    /// Distinct from `review_loop`'s own fix step in three ways: it never
4997    /// runs a reviewer wave, it never spends review-round budget, and what
4998    /// happened is recorded as an [`OperatorFixRequest`] appended to
4999    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5000    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5001    /// and vote must never be rewritten to look like a manufactured blocking
5002    /// verdict.
5003    ///
5004    /// Only meaningful once review has actually concluded: `Ready` (handed
5005    /// off with findings still open, or simply concluded clean while minor
5006    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5007    /// gate failed). Everything else is refused: a run still in progress
5008    /// should simply be resumed, and a `Merged` run's branch has already
5009    /// landed — reopening *this* run's own record cannot change that, so the
5010    /// answer there is a fresh `magi review <branch>`.
5011    ///
5012    /// A real commit here re-verifies through a fresh, ordinary review-only
5013    /// run on the same branch ([`Self::review`]) rather than reopening this
5014    /// run's own `review_loop`: once any round in this run's history went
5015    /// clean, `review_conclusion` treats that as permanent by design (the
5016    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5017    /// way to force one more genuine reviewer wave out of *this* run without
5018    /// either rewriting history or weakening that guarantee for every other
5019    /// caller. A review-only run costs nothing extra — no implementation, no
5020    /// judging, no vote — and exercises the exact same review → verify →
5021    /// gate → (human) merge path, unmodified.
5022    pub async fn fix_selected(
5023        &mut self,
5024        ids: &[String],
5025        reason: &str,
5026        allow_stale: bool,
5027    ) -> Result<()> {
5028        let reason = reason.trim();
5029        if reason.is_empty() {
5030            bail!("a fix request needs a reason — that is the operator's own record of why");
5031        }
5032        if ids.is_empty() {
5033            bail!("no finding id given");
5034        }
5035        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5036            bail!(
5037                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5038                 has already concluded — can be given a targeted fix. A run still \
5039                 in progress should simply be resumed; a `merged` run's branch has \
5040                 already landed, so its answer is a fresh `magi review <branch>`, \
5041                 not reopening this run's own record",
5042                self.state.id,
5043                self.state.status.as_str()
5044            );
5045        }
5046        let Some(winner) = self.state.winner().cloned() else {
5047            bail!("run {} has no winning candidate to fix", self.state.id);
5048        };
5049        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5050            bail!(
5051                "branch `{}` no longer exists; this run cannot be extended",
5052                winner.branch
5053            );
5054        }
5055        let home = crate::run::home();
5056        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5057            bail!(
5058                "run {} is currently being worked on by another magi process",
5059                self.state.id
5060            );
5061        }
5062        // Held for the rest of this call, including the follow-up review
5063        // below: two `magi fix` invocations against the same run must not
5064        // both reach the worktree manipulation further down, which would
5065        // otherwise race to remove and recreate the same directory — see
5066        // [`FixClaim`]'s own doc.
5067        let _claim = FixClaim::acquire(&self.state.dir())?;
5068
5069        // Resolve every id before spending anything — an unknown id refuses
5070        // the whole request rather than silently dropping it — and dedup
5071        // while keeping the operator's own order.
5072        let mut seen = BTreeSet::new();
5073        let mut findings = Vec::new();
5074        let mut missing = Vec::new();
5075        for id in ids {
5076            if !seen.insert(id.clone()) {
5077                continue;
5078            }
5079            match self.state.finding(id) {
5080                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5081                    id: f.id.clone(),
5082                    severity: f.severity,
5083                    reviewer_vote: rec.vote,
5084                    round: round.round,
5085                    round_head: round.head.clone(),
5086                    reviewer: rec.reviewer,
5087                    agent: rec.agent.clone(),
5088                    file: f.file.clone(),
5089                    line: f.line,
5090                    title: f.title.clone(),
5091                    detail: f.detail.clone(),
5092                    outcome: OperatorFixOutcome::Pending,
5093                }),
5094                None => missing.push(id.clone()),
5095            }
5096        }
5097        if !missing.is_empty() {
5098            bail!(
5099                "unknown finding id(s): {}; nothing was changed",
5100                missing.join(", ")
5101            );
5102        }
5103
5104        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5105        let stale_details: Vec<(String, String)> = findings
5106            .iter()
5107            .filter(|f| f.round_head != head_at_request)
5108            .map(|f| (f.id.clone(), f.round_head.clone()))
5109            .collect();
5110        let stale = !stale_details.is_empty();
5111        if stale && !allow_stale {
5112            bail!(
5113                "the branch has moved since some finding(s) were raised — {} — now \
5114                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5115                stale_details
5116                    .iter()
5117                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5118                    .collect::<Vec<_>>()
5119                    .join(", "),
5120                short(&head_at_request)
5121            );
5122        }
5123
5124        let request = OperatorFixRequest {
5125            requested_at: Timestamp::now(),
5126            reason: reason.to_owned(),
5127            findings,
5128            head_at_request: head_at_request.clone(),
5129            allow_stale,
5130            stale,
5131            fix: None,
5132            result_head: None,
5133            follow_up_review_run: None,
5134        };
5135        self.state.event(
5136            "fix",
5137            format!(
5138                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5139                request.findings.len(),
5140                request
5141                    .findings
5142                    .iter()
5143                    .map(|f| f.id.as_str())
5144                    .collect::<Vec<_>>()
5145                    .join(", "),
5146            ),
5147        );
5148        // Recorded now, before any worktree work or the fixer call itself —
5149        // and re-saved at each checkpoint below: a crash at any point after
5150        // this (mid fixer call, mid follow-up review) must not lose the fact
5151        // that this was requested, for which findings, and why. Everything
5152        // past this point reads and writes through `request_index` rather
5153        // than a local variable, since `request` itself is moved here.
5154        self.state.operator_fixes.push(request);
5155        self.state.save()?;
5156        let request_index = self.state.operator_fixes.len() - 1;
5157
5158        // A fresh, dedicated worktree for this one call, never the winner's
5159        // own worktree in place: that one may already be gone (folded away),
5160        // and reusing it in place would leave the branch checked out there
5161        // when the follow-up review below tries to check it out again. Freed
5162        // immediately after, either way — but only once confirmed clean:
5163        // `worktree_remove` is a `git worktree remove --force`, which would
5164        // otherwise discard uncommitted work left there by the operator or
5165        // another process before this had a chance to even look at it.
5166        let repo = self.state.repo.clone();
5167        let coordination = self.coordinate(&repo).await?;
5168        if winner.worktree.exists() {
5169            // Lockfiles a rescue commit withheld stay untracked on purpose and
5170            // are already recorded; they are not the operator's work to protect.
5171            let dirty = git::git(
5172                &winner.worktree,
5173                &["status", "--porcelain", "--untracked-files=all"],
5174            )
5175            .await?;
5176            let only_withheld = dirty.lines().all(|l| {
5177                l.strip_prefix("?? ")
5178                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5179            });
5180            if !only_withheld {
5181                bail!(
5182                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5183                     discard them first",
5184                    winner.worktree.display()
5185                );
5186            }
5187            git::worktree_remove(&self.state.repo, &winner.worktree)
5188                .await
5189                .ok();
5190        }
5191        let fix_worktree = self.state.worktree_root().join("operator-fix");
5192        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5193        git::git(
5194            &self.state.repo,
5195            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5196        )
5197        .await
5198        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5199        if !git::is_clean(&fix_worktree).await? {
5200            git::worktree_remove(&self.state.repo, &fix_worktree)
5201                .await
5202                .ok();
5203            bail!(
5204                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5205                winner.branch
5206            );
5207        }
5208        let repo_for_setup = self.state.repo.clone();
5209        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5210            Ok(()) => {
5211                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5212            }
5213            Err(e) => Err(e),
5214        };
5215        if let Err(e) = setup {
5216            git::worktree_remove(&self.state.repo, &fix_worktree)
5217                .await
5218                .ok();
5219            return Err(e);
5220        }
5221
5222        let run_id = self.state.id.clone();
5223        let prompts = self.state.config.prompts.clone();
5224        let language = self.state.config.graph.language.clone();
5225        let sessions = self.state.config.graph.sessions;
5226        let artifacts = agent::artifacts_dir(&self.state.dir());
5227        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5228            .findings
5229            .iter()
5230            .map(|f| Finding {
5231                id: f.id.clone(),
5232                severity: f.severity,
5233                file: f.file.clone(),
5234                line: f.line,
5235                title: f.title.clone(),
5236                detail: f.detail.clone(),
5237            })
5238            .collect();
5239        let fix_prompt = prompt::operator_fix_in(
5240            &self.state.instruction,
5241            &finding_list,
5242            reason,
5243            &stale_details,
5244            &head_at_request,
5245            &language,
5246            &self.state.config.graph.github_language,
5247        );
5248        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5249        drop(coordination);
5250        let (job, seat, out) = self
5251            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5252                prompt: fix_prompt.clone(),
5253                spec,
5254                seat,
5255                cwd: fix_worktree.clone(),
5256                timeout,
5257                allow_write: true,
5258                sessions,
5259                artifacts: artifacts.clone(),
5260                stem: "operator-fix".to_owned(),
5261                handover: None,
5262            })
5263            .await;
5264        let agent_id = seat.agent.clone();
5265
5266        let mut fix = FixRecord {
5267            agent: agent_id,
5268            addressed: Vec::new(),
5269            rejected: Vec::new(),
5270            notes: String::new(),
5271            committed: false,
5272            failed: None,
5273            duration_ms: 0,
5274            continuation: None,
5275        };
5276        let mut final_seat = seat.clone();
5277        match out {
5278            AgentOutcome::Ok(o) => {
5279                fix.duration_ms = o.duration_ms;
5280                let parsed = verdict::extract_json::<FixReport>(&o.text);
5281                let incomplete_reason = match &parsed {
5282                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5283                        "the reply parsed, but it reported a command whose own CLI \
5284                         never confirmed an exit status"
5285                            .to_owned(),
5286                    ),
5287                    Ok(_) => None,
5288                    Err(e) => Some(e.to_string()),
5289                };
5290                match incomplete_reason {
5291                    None => {
5292                        let report = parsed.expect("checked Ok above");
5293                        fix.addressed = report.addressed;
5294                        fix.rejected = report.rejected;
5295                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5296                    }
5297                    Some(reason) => {
5298                        let (resumed_seat, resolved, failure, cont) = self
5299                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5300                            .await;
5301                        fix.duration_ms += cont.cumulative_wait_ms;
5302                        fix.continuation = Some(cont);
5303                        final_seat = resumed_seat;
5304                        match resolved {
5305                            Some(report) => {
5306                                fix.addressed = report.addressed;
5307                                fix.rejected = report.rejected;
5308                                fix.notes =
5309                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5310                            }
5311                            None => fix.failed = failure,
5312                        }
5313                    }
5314                }
5315            }
5316            AgentOutcome::Dropped(o) => {
5317                fix.duration_ms = o.duration_ms;
5318                let why = o
5319                    .dropped
5320                    .as_ref()
5321                    .map(|d| d.why.as_str())
5322                    .unwrap_or("the CLI ended the stream without delivering its answer");
5323                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5324            }
5325            AgentOutcome::Quota(o) => {
5326                self.state.quota.push(QuotaLoss {
5327                    seat: final_seat.key.clone(),
5328                    node: "fix".to_owned(),
5329                    at: Timestamp::now(),
5330                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5331                });
5332                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5333            }
5334            AgentOutcome::Failed(e) => fix.failed = Some(e),
5335            AgentOutcome::ResourceBlocked(e) => {
5336                fix.failed = Some(format!("the fixer was not started: {e}"));
5337            }
5338        }
5339        if fix.continuation.is_none() {
5340            fix.continuation = Some(ContinuationRecord::not_needed());
5341        }
5342        self.state.seats.insert(final_seat.key.clone(), final_seat);
5343
5344        let rescue_message = format!(
5345            "magi: operator-selected fix ({}) (uncommitted work)",
5346            self.state.operator_fixes[request_index]
5347                .findings
5348                .iter()
5349                .map(|f| f.id.as_str())
5350                .collect::<Vec<_>>()
5351                .join(", ")
5352        );
5353        let repo = self.state.repo.clone();
5354        let coordination = self.coordinate(&repo).await?;
5355        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5356            self.state.note_withheld("fix", &r.withheld);
5357        }
5358        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5359        fix.committed = after != head_at_request;
5360        git::worktree_remove(&self.state.repo, &fix_worktree)
5361            .await
5362            .ok();
5363        drop(coordination);
5364
5365        self.state.event(
5366            "fix",
5367            match &fix.failed {
5368                Some(reason) => format!(
5369                    "operator fix: adoption report was lost ({reason}); {}",
5370                    if fix.committed {
5371                        "committed"
5372                    } else {
5373                        "NO new commit"
5374                    }
5375                ),
5376                None => format!(
5377                    "operator fix: {} addressed, {} rejected, {}",
5378                    fix.addressed.len(),
5379                    fix.rejected.len(),
5380                    if fix.committed {
5381                        "committed"
5382                    } else {
5383                        "NO new commit"
5384                    }
5385                ),
5386            },
5387        );
5388
5389        // Every selected finding gets an outcome — never left `Pending` once
5390        // the fixer's own turn is over. A report that never came back at all
5391        // marks every one of them `Unreported`, not silently "not addressed":
5392        // quota, a dropped stream, or an exhausted continuation are gaps in
5393        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5394        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5395        for f in &mut self.state.operator_fixes[request_index].findings {
5396            f.outcome = if fix.failed.is_some() {
5397                OperatorFixOutcome::Unreported
5398            } else if fix.addressed.contains(&f.id) {
5399                OperatorFixOutcome::Addressed
5400            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5401                OperatorFixOutcome::Rejected { why: r.why.clone() }
5402            } else {
5403                OperatorFixOutcome::Unreported
5404            };
5405        }
5406
5407        let committed = fix.committed;
5408        if committed {
5409            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5410        }
5411        self.state.operator_fixes[request_index].fix = Some(fix);
5412        // Saved again now that the fixer's own outcome is final, on top of
5413        // the save right after the request was first pushed above.
5414        self.state.save()?;
5415
5416        if committed {
5417            self.state.event(
5418                "fix",
5419                format!(
5420                    "operator fix committed {}; opening a follow-up review-only run",
5421                    short(&after)
5422                ),
5423            );
5424            // The operator asked for the fix, and the follow-up serves whatever
5425            // task the run it follows served.
5426            let origin =
5427                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5428            match Self::review(
5429                &self.state.repo,
5430                &winner.branch,
5431                self.state.config.clone(),
5432                origin,
5433            )
5434            .await
5435            {
5436                Ok(mut follow_up) => {
5437                    follow_up.state.event(
5438                        "start",
5439                        format!(
5440                            "requested by an operator fix on run {} for finding(s) {}",
5441                            self.state.id,
5442                            self.state.operator_fixes[request_index]
5443                                .findings
5444                                .iter()
5445                                .map(|f| f.id.as_str())
5446                                .collect::<Vec<_>>()
5447                                .join(", "),
5448                        ),
5449                    );
5450                    follow_up.state.save()?;
5451                    let follow_up_id = follow_up.state.id.clone();
5452                    // The follow-up is a run like any other: it belongs to the
5453                    // task of the run it follows, or to one filed for it.
5454                    let adopted = match crate::direct::adopt(
5455                        &follow_up.state,
5456                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5457                    ) {
5458                        Ok(a) => a,
5459                        Err(e) => {
5460                            // An ownerless run must not spend agent calls; it
5461                            // stays saved, and `magi run --resume` adopts it.
5462                            self.state.event(
5463                                "fix",
5464                                format!(
5465                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5466                                ),
5467                            );
5468                            self.state.save()?;
5469                            return Ok(());
5470                        }
5471                    };
5472                    let executed = follow_up.execute().await;
5473                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5474                    if let Some(a) = adopted {
5475                        a.finish(&follow_up.state, executed);
5476                    }
5477                    if let Some(e) = failure {
5478                        self.state.event(
5479                            "fix",
5480                            format!(
5481                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5482                            ),
5483                        );
5484                    }
5485                    self.state.operator_fixes[request_index].follow_up_review_run =
5486                        Some(follow_up_id);
5487                }
5488                Err(e) => {
5489                    self.state.event(
5490                        "fix",
5491                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5492                    );
5493                }
5494            }
5495            self.state.save()?;
5496        }
5497
5498        Ok(())
5499    }
5500
5501    // --------------------------------------------------------------- review
5502
5503    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5504    /// names, in order, each at most once, moving on only when the call
5505    /// advances (an error, a quota hit or nothing usable - the same decision
5506    /// point `agent::chain_advances` is for the other chained roles).
5507    ///
5508    /// Each agent gets its own seat from `Runner::seat`: the same agent
5509    /// continues its conversation (the winner's own implementer seat when it
5510    /// is the winner's author, now that the competition is over), another
5511    /// takes a fresh one so the full prompt is sent again. A handover is
5512    /// recorded under `node`, which is what makes the fallback stick for the
5513    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5514    /// is returned, so an exhausted chain reads exactly like a single failed
5515    /// fixer: one quota loss, the same wording, the same refund. An earlier
5516    /// attempt's edits are left in the tree and judged, with the final
5517    /// attempt's, by what git says afterwards.
5518    ///
5519    /// Returns the job that produced the outcome, for `continue_fix_report`.
5520    async fn ask_fixer(
5521        &mut self,
5522        winner: &Candidate,
5523        node: &'static str,
5524        round: Option<usize>,
5525        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5526    ) -> (SeatJob, SeatState, AgentOutcome) {
5527        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5528        self.ask_fixer_among(attempts, node, round, None, build)
5529            .await
5530    }
5531
5532    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5533    /// worktree and a commit: before the next agent is asked after one that
5534    /// did not satisfy the call, the worktree is put back on that commit, so
5535    /// a failed agent's partial edits are not carried into the next one.
5536    async fn ask_fixer_among(
5537        &mut self,
5538        attempts: Vec<(AgentSpec, String)>,
5539        node: &'static str,
5540        round: Option<usize>,
5541        restore: Option<(&Path, &str)>,
5542        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5543    ) -> (SeatJob, SeatState, AgentOutcome) {
5544        let run_id = self.state.id.clone();
5545        let prompts = self.state.config.prompts.clone();
5546        let cache = self.state.config.cache_dir();
5547        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5548        let mut last = None;
5549        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5550            let seat = self.seat(&key, &spec.id);
5551            let mut job = build(spec.clone(), seat);
5552            if i > 0 {
5553                job.stem = format!("{}-{}", job.stem, spec.id);
5554            }
5555            let ctx = WaveCtx {
5556                carry_seats: false,
5557                pauses: Some((&self.pause, &self.interrupt)),
5558                run: &run_id,
5559                node,
5560                prompts: &prompts,
5561                cache: cache.as_deref(),
5562                round,
5563            };
5564            let (seat, out) =
5565                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5566            // A resource block never advances the chain: the chosen agent
5567            // was not asked, so it must be the one asked again on resume.
5568            let advances = match &out {
5569                AgentOutcome::Ok(o) => agent::output_advances(o),
5570                AgentOutcome::ResourceBlocked(_) => false,
5571                _ => true,
5572            };
5573            if let Some(next) = ids.get(i + 1)
5574                && advances
5575            {
5576                self.state.seats.insert(seat.key.clone(), seat.clone());
5577                let class =
5578                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5579                let reason = match &out {
5580                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5581                    other => fail_reason(other),
5582                };
5583                // A failed agent's edits must not reach the next one, and a
5584                // restore that did not work ends the chain: the call fails
5585                // and the caller's own (confirmed) restore decides.
5586                if let Some((wt, rev)) = restore
5587                    && let Err(e) = self.graft_restore(wt, rev).await
5588                {
5589                    let why =
5590                        format!("could not restore the worktree before the next fixer: {e:#}");
5591                    tracing::warn!("{why}");
5592                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5593                        AgentOutcome::ResourceBlocked(why)
5594                    } else {
5595                        AgentOutcome::Failed(why)
5596                    };
5597                    last = Some((job, seat, out));
5598                    break;
5599                }
5600                record_handover(
5601                    &mut self.state,
5602                    node,
5603                    &seat.key,
5604                    &spec.id,
5605                    next,
5606                    &class,
5607                    &reason,
5608                );
5609                continue;
5610            }
5611            last = Some((job, seat, out));
5612            break;
5613        }
5614        last.expect("the fixer chain always has an entry")
5615    }
5616
5617    /// The one repair a verification-only failure earns (see
5618    /// [`crate::recovery`]), before the first review round of a run that took
5619    /// a branch over.
5620    ///
5621    /// Saved as started *before* the fixer is called, so a resume never
5622    /// repairs twice; a start found on re-entry is rolled back to the commit
5623    /// it began from. A failed or unusable fixer leaves the tree as it was and
5624    /// the review goes on over it, so the first round's own verification
5625    /// reports the failure again. The failure is never carried into a round as
5626    /// a result: verification is re-run on whatever commit this leaves.
5627    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5628        use crate::recovery::{Mode, Repair};
5629        if !self.state.reviews.is_empty() {
5630            return Ok(());
5631        }
5632        let Some(mut rec) = self.state.recovery.clone() else {
5633            return Ok(());
5634        };
5635        if rec.mode != Mode::VerifyRepair {
5636            return Ok(());
5637        }
5638        let wt = winner.worktree.clone();
5639        let mut head = git::rev_parse(&wt, "HEAD").await?;
5640        match rec.repair.clone() {
5641            Repair::Done { .. } => return Ok(()),
5642            Repair::WaitingForRestore { before } => {
5643                self.graft_restore(&wt, &before).await?;
5644                head = before;
5645                rec.repair = Repair::NotStarted;
5646                self.state.recovery = Some(rec.clone());
5647                self.state.save()?;
5648            }
5649            Repair::Started { before } => {
5650                // Interrupted: the unfinished edits are not trusted.
5651                // An unconfirmed rollback blocks the run with `Started` kept,
5652                // so a resume retries it; the tree is never reviewed as is.
5653                if let Err(e) = self.graft_restore(&wt, &before).await {
5654                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5655                        return Err(e);
5656                    }
5657                    return self
5658                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5659                        .await;
5660                }
5661                rec.repair = Repair::Done {
5662                    before: before.clone(),
5663                    after: before,
5664                    committed: false,
5665                    failed: Some("interrupted; rolled back".to_owned()),
5666                };
5667                self.state
5668                    .event("recovery", "the repair was interrupted and rolled back");
5669                self.state.recovery = Some(rec);
5670                self.state.save()?;
5671                return Ok(());
5672            }
5673            Repair::NotStarted => {}
5674        }
5675        if crate::recovery::recheck(&mut rec, &head) {
5676            self.state.event("recovery", rec.summary());
5677            self.state.recovery = Some(rec);
5678            self.state.save()?;
5679            return Ok(());
5680        }
5681        rec.repair = Repair::Started {
5682            before: head.clone(),
5683        };
5684        self.state.recovery = Some(rec.clone());
5685        self.state.save()?;
5686
5687        let language = self.state.config.graph.language.clone();
5688        let sessions = self.state.config.graph.sessions;
5689        let artifacts = agent::artifacts_dir(&self.state.dir());
5690        let fix_prompt = prompt::recovery_fix(
5691            &self.state.instruction,
5692            &rec,
5693            self.state.config.graph.review_rounds,
5694            &language,
5695        );
5696        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5697        self.state.event(
5698            "recovery",
5699            format!(
5700                "repair: the fixer starts from the failed verification on {}",
5701                short(&head)
5702            ),
5703        );
5704        let (_job, seat, out) = self
5705            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5706                prompt: fix_prompt.clone(),
5707                spec,
5708                seat,
5709                cwd: wt.clone(),
5710                timeout,
5711                allow_write: true,
5712                sessions,
5713                artifacts: artifacts.clone(),
5714                stem: "fix-recovery".to_owned(),
5715                handover: None,
5716            })
5717            .await;
5718        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5719            rec.repair = Repair::WaitingForRestore { before: head };
5720            self.state.recovery = Some(rec);
5721            return self.stop_for_resource();
5722        }
5723        let failed = match &out {
5724            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5725            AgentOutcome::Quota(o) => {
5726                self.state.quota.push(QuotaLoss {
5727                    seat: seat.key.clone(),
5728                    node: "fix".to_owned(),
5729                    at: Timestamp::now(),
5730                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5731                });
5732                Some("rate limited (quota); the fixer could not run".to_owned())
5733            }
5734            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5735            AgentOutcome::Failed(e) => Some(e.clone()),
5736        };
5737        self.state.seats.insert(seat.key.clone(), seat);
5738        let coordination = self.coordinate(&wt).await?;
5739        let failed = match failed {
5740            // The agent's partial edits are not kept.
5741            Some(why) => {
5742                if let Err(e) = git::restore_to(&wt, &head).await {
5743                    return self
5744                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5745                        .await;
5746                }
5747                Some(why)
5748            }
5749            None => {
5750                if let Ok(r) = git::rescue_commit(
5751                    &wt,
5752                    "magi: recovery repair of the failed verification (uncommitted work)",
5753                )
5754                .await
5755                {
5756                    self.state.note_withheld("fix", &r.withheld);
5757                }
5758                None
5759            }
5760        };
5761        let after = git::rev_parse(&wt, "HEAD").await?;
5762        drop(coordination);
5763        self.state.event(
5764            "recovery",
5765            match &failed {
5766                Some(why) => format!("repair: nothing kept ({why})"),
5767                None if after != head => {
5768                    format!("repair: committed {}", short(&after))
5769                }
5770                None => "repair: the fixer made no commit".to_owned(),
5771            },
5772        );
5773        rec.repair = Repair::Done {
5774            before: head.clone(),
5775            committed: after != head,
5776            after,
5777            failed,
5778        };
5779        self.state.recovery = Some(rec);
5780        self.state.save()?;
5781        Ok(())
5782    }
5783
5784    /// A repair whose rollback could not be confirmed: the run is blocked and
5785    /// resumable, and the unconfirmed state is never reviewed.
5786    async fn stop_recovery(
5787        &mut self,
5788        mut rec: crate::recovery::RecoveryRecord,
5789        head: &str,
5790        why: String,
5791    ) -> Result<()> {
5792        // `Started` stays recorded: a resume retries the restore.
5793        rec.repair = crate::recovery::Repair::Started {
5794            before: head.to_owned(),
5795        };
5796        self.state.recovery = Some(rec);
5797        self.state
5798            .event("recovery", format!("repair could not be undone: {why}"));
5799        self.state.status = RunStatus::Blocked;
5800        self.state.save()?;
5801        bail!("recovery repair could not be rolled back: {why}")
5802    }
5803
5804    async fn review_loop(&mut self) -> Result<()> {
5805        // A base that would not rebase is a person's decision, not a review
5806        // round: nothing here would change the answer, and reviewers and a
5807        // fixer would be spending real budget on a tree that cannot land
5808        // regardless of what they find.
5809        if self
5810            .state
5811            .base_sync
5812            .as_ref()
5813            .is_some_and(|s| s.conflict.is_some())
5814        {
5815            return Ok(());
5816        }
5817        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5818        // agent files with `magi task add` name the run that paid for it. The
5819        // prompt overlay is cloned alongside it because the waves borrow it
5820        // while `self` is mutably borrowed by the node's own bookkeeping.
5821        let run_id = self.state.id.clone();
5822        let prompts = self.state.config.prompts.clone();
5823        let Some(winner) = self.state.winner().cloned() else {
5824            return Ok(());
5825        };
5826        let max_rounds = self.state.config.graph.review_rounds;
5827        // A clean round, an exhausted round budget, or a stalled tree (see
5828        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5829        // they are recorded — recomputed here, not read off `status`, so a
5830        // reentry into a run that already stopped restates the identical
5831        // verdict instead of silently handing back whatever an earlier node
5832        // in this same walk clobbered `status` to (a solo-candidate
5833        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5834        // below runs an empty range once the budget is spent, and would
5835        // otherwise fall through without touching `status` at all.
5836        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5837            // A reentry after a crash between the last round's save and
5838            // `stop_reviewing` reaches the hand-off here, not there.
5839            if status == RunStatus::Gating {
5840                self.record_contested_handoff();
5841            }
5842            self.state.status = status;
5843            self.state.save()?;
5844            return Ok(());
5845        }
5846        self.state.status = RunStatus::Reviewing;
5847        // A last recorded round whose own verification never resolved
5848        // (`ResourceBlocked` — the shared build cache, not the patch) is
5849        // never a concluded round, whatever the round budget says: starting
5850        // a fresh round on top of it would spend a whole new reviewer wave
5851        // re-reading an unchanged patch instead of just retrying the one
5852        // check that actually needs it, and once the budget is spent the
5853        // loop below has nothing left to do at all (its range is empty).
5854        // Retry that check directly instead, exactly the same retry
5855        // `stop_reviewing` already does for its own catch-up case.
5856        if self
5857            .state
5858            .reviews
5859            .last()
5860            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5861        {
5862            let shell = self.state.config.shell();
5863            return self
5864                .stop_reviewing(
5865                    "the last round's own verification never resolved",
5866                    &shell,
5867                    &winner.worktree,
5868                )
5869                .await;
5870        }
5871
5872        self.recovery_repair(&winner).await?;
5873        if self.resource_halt()? {
5874            return Ok(());
5875        }
5876
5877        let repo = self.state.repo.clone();
5878        let root = self.state.worktree_root();
5879        let language = self.state.config.graph.language.clone();
5880        let sessions = self.state.config.graph.sessions;
5881        let artifacts = self.merge_repair_artifacts();
5882        let base = self.landing_base();
5883        let base_short = short(&base);
5884        let reviewers = self.roles.reviewers.clone();
5885        let shell = self.state.config.shell();
5886
5887        for round in (self.state.reviews.len() + 1)..=max_rounds {
5888            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5889            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5890            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5891            // The prior round's own record, already persisted — never a
5892            // hand-carried variable of just its failing output: that is
5893            // exactly what let a round's e2e result drift out of sync with
5894            // which commit it was actually about (see `SCHEMA`'s doc for
5895            // schema 8). Judged against `head`, the commit reviewers are
5896            // about to look at now, so the summary always reads as "an
5897            // earlier head" here — this round's own patch has not been
5898            // checked yet.
5899            let prev_verification = self
5900                .state
5901                .reviews
5902                .last()
5903                .and_then(|r| r.verification_summary(&head));
5904            // Only the first round of a run that took a branch over: later
5905            // rounds have this run's own history.
5906            let prior_section = match (&self.state.recovery, round) {
5907                (Some(rec), 1) if self.state.reviews.is_empty() => {
5908                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5909                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5910                            Some(crate::recovery::head_of(
5911                                &git::diff(&winner.worktree, reviewed, "HEAD")
5912                                    .await
5913                                    .unwrap_or_default(),
5914                                crate::recovery::SINCE_DIFF_MAX,
5915                            ))
5916                        }
5917                        _ => None,
5918                    };
5919                    Some(prompt::recovery_review(rec, since.as_deref()))
5920                }
5921                _ => None,
5922            };
5923
5924            // Each reviewer gets its own detached checkout of exactly this
5925            // commit: nobody can perturb the winner's tree, and the fixer can
5926            // keep working without racing a reviewer.
5927            let mut jobs = Vec::new();
5928            let coordination = self.coordinate(&repo).await?;
5929            for (r, spec) in reviewers.iter().cloned().enumerate() {
5930                let wt = root.join(format!("review-{}", r + 1));
5931                // `clean -fdx` in a reset wipes setup's products, so every
5932                // round sets the seat up again.
5933                if wt.exists() {
5934                    git::reset_detached(&wt, &head).await?;
5935                } else {
5936                    git::worktree_add_detached(&repo, &wt, &head).await?;
5937                }
5938                self.setup_seat_worktree(&repo, &wt, false).await?;
5939                let seat_key = format!("review-{}", r + 1);
5940                // The seat starts the round on whoever answered it last, not
5941                // on the agent the spec names, so a failure is not re-paid.
5942                let spec = pick_start_spec(
5943                    &self.roles.reviewer_roster,
5944                    spec,
5945                    self.state.seat_history.get(&seat_key),
5946                );
5947                let seat = self.seat(&seat_key, &spec.id);
5948                jobs.push(SeatJob {
5949                    prompt: prompt::review(&prompt::ReviewCtx {
5950                        instruction: &self.state.instruction,
5951                        branch: &winner.branch,
5952                        base_short: &base_short,
5953                        stat: &stat,
5954                        patch: &patch,
5955                        verification: prev_verification.as_ref(),
5956                        prior: prior_section.as_deref(),
5957                        reviewers: reviewers.len(),
5958                        round,
5959                        rounds: max_rounds,
5960                        // A review-only run has no rankings, so nothing
5961                        // competed for this patch and the reviewer is told so.
5962                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5963                        lens: Lens::for_seat(r),
5964                        language: &language,
5965                        github_language: &self.state.config.graph.github_language,
5966                    }),
5967                    spec,
5968                    seat,
5969                    cwd: wt,
5970                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5971                    allow_write: false,
5972                    sessions,
5973                    artifacts: artifacts.clone(),
5974                    stem: format!("review-{round}-{}", r + 1),
5975                    handover: None,
5976                });
5977            }
5978
5979            drop(coordination);
5980            self.state.event(
5981                "review",
5982                format!(
5983                    "round {round}: {} reviewers on {}",
5984                    jobs.len(),
5985                    short(&head)
5986                ),
5987            );
5988            let mut quota_losses = Vec::new();
5989            let review_retries = self.state.config.graph.retries;
5990            let review_cache = self.state.config.cache_dir();
5991            let ctx = WaveCtx {
5992                carry_seats: true,
5993                pauses: Some((&self.pause, &self.interrupt)),
5994                run: &run_id,
5995                node: "review",
5996                prompts: &prompts,
5997                cache: review_cache.as_deref(),
5998                round: Some(round),
5999            };
6000            let results = ask_json_wave::<Review>(
6001                jobs,
6002                Arc::clone(&self.sem),
6003                review_retries,
6004                &self.roles.reviewer_roster,
6005                &ctx,
6006                &mut quota_losses,
6007                &mut self.state,
6008                &|_: &Review| Ok(()),
6009            )
6010            .await;
6011            if self.state.resource_block.is_some() {
6012                return self.stop_for_resource();
6013            }
6014            // Counted before the move below: how many of *this* round's
6015            // reviewer seats were lost to their own rate limit, as opposed to
6016            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6017            let round_quota_missing = quota_losses.len();
6018            self.state.quota.extend(quota_losses);
6019
6020            let mut records = Vec::new();
6021            let mut all_findings = Vec::new();
6022            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6023                let agent_id = seat.agent.clone();
6024                self.state.seats.insert(seat.key.clone(), seat);
6025                let mut record = ReviewRecord {
6026                    reviewer: r + 1,
6027                    agent: agent_id,
6028                    summary: String::new(),
6029                    findings: Vec::new(),
6030                    vote: None,
6031                    failed: None,
6032                    duration_ms: 0,
6033                    // Set for both outcomes: `failed: Some(_)` with
6034                    // `attempts > 0` is a seat every retry still lost, not a
6035                    // recovered one — only `failed: None` with `attempts > 0`
6036                    // reads as "answered after a nudge" (see this field's own
6037                    // doc).
6038                    attempts,
6039                };
6040                match res {
6041                    Ok((review, out)) => {
6042                        // Sanitized here, at the point every other piece of
6043                        // agent prose in this file is (candidate summaries,
6044                        // deliberation turns, vote reasons): a reviewer's own
6045                        // words are the one thing about it that could name
6046                        // it, and reconsideration below broadcasts this same
6047                        // summary and these same findings to every other
6048                        // seat on the panel.
6049                        record.summary =
6050                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6051                        record.vote = Some(review.vote);
6052                        record.duration_ms = out.duration_ms;
6053                        for (n, mut f) in review.findings.into_iter().enumerate() {
6054                            // ids are magi's, never the agent's: the fixer's
6055                            // adoption report is keyed by them.
6056                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6057                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6058                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6059                            // `file` is agent-supplied prose too, never
6060                            // checked against the real tree — the same
6061                            // exposure `title`/`detail` above have, just in
6062                            // a field easy to forget because it looks like a
6063                            // path rather than free text.
6064                            f.file = f
6065                                .file
6066                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6067                            all_findings.push(f.clone());
6068                            record.findings.push(f);
6069                        }
6070                        self.state.event(
6071                            "review",
6072                            format!(
6073                                "round {round}: reviewer {} voted {} with {} finding(s)",
6074                                r + 1,
6075                                review.vote.label(),
6076                                record.findings.len()
6077                            ),
6078                        );
6079                    }
6080                    Err(e) => {
6081                        record.failed = Some(e.to_string());
6082                        self.state.event(
6083                            "review",
6084                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6085                        );
6086                    }
6087                }
6088                records.push(record);
6089            }
6090
6091            // Tally the round's votes and, if they split, spend the one
6092            // round of reconsideration the split -> deliberate -> revote
6093            // shape `judge`/`vote` use for the panel, sized down to what a
6094            // read-only review round can afford: one round, and a revote
6095            // rather than an argument, because the panel already wrote its
6096            // reasoning down as findings the first time around.
6097            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6098            let vote_split =
6099                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6100            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6101            if vote_split {
6102                self.state.event(
6103                    "review",
6104                    format!(
6105                        "round {round}: votes split ({}) — one round of reconsideration",
6106                        initial_votes
6107                            .iter()
6108                            .map(|v| v.label())
6109                            .collect::<Vec<_>>()
6110                            .join(", ")
6111                    ),
6112                );
6113                // Seats read every seat's findings and votes, still numbered
6114                // and never named — the same anonymity `review` itself keeps.
6115                let panel: Vec<ReviewSeatReport<'_>> = records
6116                    .iter()
6117                    .filter_map(|r| {
6118                        r.vote.map(|vote| ReviewSeatReport {
6119                            reviewer: r.reviewer,
6120                            vote,
6121                            summary: &r.summary,
6122                            findings: &r.findings,
6123                        })
6124                    })
6125                    .collect();
6126
6127                let mut jobs = Vec::new();
6128                let mut seats_at = Vec::new();
6129                for (r, spec) in reviewers.iter().cloned().enumerate() {
6130                    // A seat with no initial vote has nothing to reconsider
6131                    // from and stays absent, the same as it stayed absent
6132                    // from `panel` above.
6133                    if records[r].vote.is_none() {
6134                        continue;
6135                    }
6136                    let wt = root.join(format!("review-{}", r + 1));
6137                    let seat_key = format!("review-{}", r + 1);
6138                    let spec = self.occupant(&seat_key, spec);
6139                    let seat = self.seat(&seat_key, &spec.id);
6140                    // A seat with no live session has already forgotten the
6141                    // initial review's prompt — restate the patch it is
6142                    // voting on, the same as `deliberate`/`vote` do for a
6143                    // judge in the same position.
6144                    // The panel already carries this seat's own review and
6145                    // vote, so restating the patch makes the prompt whole for
6146                    // a seat handed to another agent.
6147                    let build = |with_patch: bool| {
6148                        prompt::review_reconsider(&ReviewReconsiderCtx {
6149                            instruction: &self.state.instruction,
6150                            reviewer: r + 1,
6151                            lens: Lens::for_seat(r),
6152                            panel: &panel,
6153                            patch: with_patch.then_some(ReviewPatch {
6154                                branch: &winner.branch,
6155                                base_short: &base_short,
6156                                stat: &stat,
6157                                patch: &patch,
6158                            }),
6159                            round,
6160                            rounds: max_rounds,
6161                            language: &language,
6162                        })
6163                    };
6164                    let full = build(true);
6165                    let prompt = if has_context(&spec, &seat, sessions) {
6166                        build(false)
6167                    } else {
6168                        full.clone()
6169                    };
6170                    jobs.push(SeatJob {
6171                        prompt,
6172                        spec,
6173                        seat,
6174                        cwd: wt,
6175                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6176                        allow_write: false,
6177                        sessions,
6178                        artifacts: artifacts.clone(),
6179                        stem: format!("review-{round}-reconsider-{}", r + 1),
6180                        handover: Some(full),
6181                    });
6182                    seats_at.push(r);
6183                }
6184
6185                let mut recon_quota_losses = Vec::new();
6186                let recon_cache = self.state.config.cache_dir();
6187                let recon_ctx = WaveCtx {
6188                    carry_seats: true,
6189                    pauses: Some((&self.pause, &self.interrupt)),
6190                    run: &run_id,
6191                    node: "review",
6192                    prompts: &prompts,
6193                    cache: recon_cache.as_deref(),
6194                    round: Some(round),
6195                };
6196                let recon_results = ask_json_wave::<ReviewRevote>(
6197                    jobs,
6198                    Arc::clone(&self.sem),
6199                    review_retries,
6200                    &self.roles.reviewer_roster,
6201                    &recon_ctx,
6202                    &mut recon_quota_losses,
6203                    &mut self.state,
6204                    &|_: &ReviewRevote| Ok(()),
6205                )
6206                .await;
6207                if self.state.resource_block.is_some() {
6208                    return self.stop_for_resource();
6209                }
6210                self.state.quota.extend(recon_quota_losses);
6211
6212                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6213                    let agent_id = seat.agent.clone();
6214                    self.state.seats.insert(seat.key.clone(), seat);
6215                    let mut rec = ReviewRevoteRecord {
6216                        reviewer: r + 1,
6217                        agent: agent_id,
6218                        vote: None,
6219                        reason: String::new(),
6220                        failed: None,
6221                    };
6222                    match res {
6223                        Ok((rv, _)) => {
6224                            rec.vote = Some(rv.vote);
6225                            rec.reason =
6226                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6227                            self.state.event(
6228                                "review",
6229                                format!(
6230                                    "round {round}: reviewer {} revoted {}",
6231                                    r + 1,
6232                                    rv.vote.label()
6233                                ),
6234                            );
6235                        }
6236                        Err(e) => {
6237                            rec.failed = Some(e.to_string());
6238                            self.state.event(
6239                                "review",
6240                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6241                            );
6242                        }
6243                    }
6244                    reconsideration.push(rec);
6245                }
6246            } else if initial_votes.len() > 1 {
6247                self.state.event(
6248                    "review",
6249                    format!(
6250                        "round {round}: votes agreed ({}) — no reconsideration",
6251                        initial_votes[0].label()
6252                    ),
6253                );
6254            }
6255
6256            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6257            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6258            // A round that already has a blocking finding and a round left to
6259            // try is going back to the fixer no matter what `verify.e2e`
6260            // says, so running it first only spends the loop's slowest step
6261            // (minutes, for a Rust repo's full test suite) on a head about
6262            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6263            // runs once a round has no blocking findings left (see
6264            // `round_is_clean`, which a deferred — empty — `e2e` can never
6265            // satisfy since `blocking` is nonzero whenever this branch is
6266            // taken), and `stop_reviewing` forces a real run before it will
6267            // ever read a deferred round as green.
6268            let defer_e2e =
6269                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6270            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6271                let reason =
6272                    format!("{blocking} blocking finding(s) already required a fix this round");
6273                self.state.event(
6274                    "verify",
6275                    format!(
6276                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6277                         {}); it will run once a round has none left",
6278                        short(&head)
6279                    ),
6280                );
6281                (Vec::new(), false, true, Some(reason))
6282            } else {
6283                let e2e_commands = self.state.config.verify.e2e.clone();
6284                let cache_dir = self.state.config.cache_dir();
6285                let context = format!("round {round}");
6286                let wait = self.repository_wait();
6287                let (e2e, verify_retried) = with_cache_lease(
6288                    &mut self.state,
6289                    cache_dir.as_deref(),
6290                    "e2e",
6291                    "e2e",
6292                    &winner.worktree,
6293                    &head,
6294                    verify_timeout,
6295                    &context,
6296                    &wait,
6297                    |state, budget| {
6298                        let shell = shell.clone();
6299                        let e2e_commands = e2e_commands.clone();
6300                        let worktree = winner.worktree.clone();
6301                        let context = context.clone();
6302                        async move {
6303                            run_e2e_with_retry(
6304                                state,
6305                                &shell,
6306                                &e2e_commands,
6307                                &worktree,
6308                                budget,
6309                                &context,
6310                            )
6311                            .await
6312                        }
6313                    },
6314                )
6315                .await?;
6316                (e2e, verify_retried, false, None)
6317            };
6318
6319            let expected = records.len();
6320            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6321            let incomplete = answered < expected;
6322            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6323            let policy = self.state.config.graph.incomplete_review;
6324            let clean = round_is_clean(
6325                blocking,
6326                e2e_ok,
6327                answered,
6328                expected,
6329                round_quota_missing,
6330                policy,
6331            );
6332
6333            let mut round_record = ReviewRound {
6334                round,
6335                head: head.clone(),
6336                verified_head: None,
6337                verified_at: None,
6338                reviews: records,
6339                e2e,
6340                verify_retried,
6341                e2e_deferred,
6342                e2e_defer_reason,
6343                fix: None,
6344                blocking,
6345                answered,
6346                expected,
6347                clean,
6348                progressed: false,
6349                vote_split,
6350                reconsideration,
6351                verdict: None,
6352            };
6353            // The final vote per seat is its revote where reconsideration
6354            // ran and answered, its initial vote otherwise — the same
6355            // fallback `tally` uses for a judge whose private vote failed.
6356            round_record.verdict = ReviewVote::worst(
6357                round_record
6358                    .final_votes()
6359                    .into_iter()
6360                    .map(|(_, _, vote)| vote),
6361            );
6362            // Which commit and when magi actually attempted to check —
6363            // known the moment a command was dispatched against `head`,
6364            // whether or not it finished: a resource-blocked attempt still
6365            // targeted a specific commit at a specific time, and leaving
6366            // that unrecorded is exactly what made `verification_summary`
6367            // report a fresh attempt as "commit unknown ... recorded before
6368            // this was tracked", indistinguishable from a genuinely old,
6369            // untracked record. Only a deferred or unconfigured round never
6370            // ran at all and has nothing to record — see
6371            // `ReviewRound::verified_head`'s own doc.
6372            if !matches!(
6373                round_record.e2e_status(),
6374                E2eStatus::Deferred | E2eStatus::NotConfigured
6375            ) {
6376                round_record.verified_head = Some(head.clone());
6377                round_record.verified_at = Some(Timestamp::now());
6378            }
6379            let this_round_verification = round_record.verification_summary(&head);
6380
6381            if incomplete {
6382                let missing: Vec<String> = round_record
6383                    .reviews
6384                    .iter()
6385                    .filter(|r| r.failed.is_some())
6386                    .map(|r| format!("review-{}", r.reviewer))
6387                    .collect();
6388                self.state.event(
6389                    "review",
6390                    format!(
6391                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6392                        missing.join(", ")
6393                    ),
6394                );
6395            }
6396
6397            if clean {
6398                self.state.event(
6399                    "review",
6400                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6401                        format!(
6402                            "round {round}: clean (warn policy, incomplete panel) — no \
6403                             blocking findings from the seats that answered, verification green"
6404                        )
6405                    } else if incomplete {
6406                        format!(
6407                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6408                             quorum) — no blocking findings from the seats that answered, \
6409                             verification green",
6410                            expected - answered
6411                        )
6412                    } else {
6413                        format!("round {round}: clean — no blocking findings, verification green")
6414                    },
6415                );
6416                self.state.reviews.push(round_record);
6417                self.state.status = RunStatus::Gating;
6418                self.state.save()?;
6419                return Ok(());
6420            }
6421
6422            // Nothing was raised and verification passed, but not every seat
6423            // answered and `round_is_clean` still refused to call it clean —
6424            // either a seat is missing for a reason other than its own quota
6425            // (a crash, a timeout, unparsable output — worth another try), or
6426            // every seat that could have answered lost its quota and nobody
6427            // is left to decide on: re-review rather than send the fixer
6428            // after a round with nothing to fix.
6429            if incomplete && blocking == 0 && e2e_ok {
6430                self.state.reviews.push(round_record);
6431                self.state.save()?;
6432                if round == max_rounds {
6433                    self.state.status = RunStatus::Blocked;
6434                    self.state.event(
6435                        "review",
6436                        format!(
6437                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6438                             refusing to call it clean",
6439                            expected - answered
6440                        ),
6441                    );
6442                    return Ok(());
6443                }
6444                continue;
6445            }
6446
6447            // Nothing for the fixer to act on (`blocking == 0`) and the only
6448            // reason this round is not clean is that magi itself never got
6449            // a command to run — the shared build cache, not the patch (see
6450            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6451            // the fixer would invite a change to appease contention that has
6452            // nothing to do with the diff, and would leave this attempt
6453            // sitting in the next round's prompt as if it were about an
6454            // earlier, superseded commit rather than what it actually is:
6455            // the same head, still waiting to be checked. Wait for it the
6456            // same way the final round's own contention is already handled,
6457            // whatever round this happens to be.
6458            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6459                self.state.reviews.push(round_record);
6460                return self
6461                    .stop_reviewing(
6462                        "the round's own verification could not run",
6463                        &shell,
6464                        &winner.worktree,
6465                    )
6466                    .await;
6467            }
6468
6469            if round == max_rounds {
6470                self.state.reviews.push(round_record);
6471                return self
6472                    .stop_reviewing(
6473                        &format!(
6474                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6475                        ),
6476                        &shell,
6477                        &winner.worktree,
6478                    )
6479                    .await;
6480            }
6481
6482            // Fix. The winner's own implementer seat continues its conversation:
6483            // the competition is over, so context is pure benefit now.
6484            let blocking_findings: Vec<_> = all_findings
6485                .iter()
6486                .filter(|f| f.severity.blocks())
6487                .cloned()
6488                .collect();
6489            let fix_prompt = prompt::fix_in(
6490                &self.state.instruction,
6491                &blocking_findings,
6492                this_round_verification.as_ref(),
6493                round,
6494                max_rounds,
6495                &language,
6496                &self.state.config.graph.github_language,
6497            );
6498            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6499            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6500            let (job, seat, out) = self
6501                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6502                    prompt: fix_prompt.clone(),
6503                    spec,
6504                    seat,
6505                    cwd: winner.worktree.clone(),
6506                    timeout,
6507                    allow_write: true,
6508                    sessions,
6509                    artifacts: artifacts.clone(),
6510                    stem: format!("fix-{round}"),
6511                    handover: None,
6512                })
6513                .await;
6514            // Not asked: the round is not recorded (so a resume repeats it),
6515            // no fixer was handed over to, and the seat is left as it was.
6516            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6517                return self.stop_for_resource();
6518            }
6519            let agent_id = seat.agent.clone();
6520
6521            let mut fix = FixRecord {
6522                agent: agent_id,
6523                addressed: Vec::new(),
6524                rejected: Vec::new(),
6525                notes: String::new(),
6526                committed: false,
6527                failed: None,
6528                duration_ms: 0,
6529                continuation: None,
6530            };
6531            let mut continuation = ContinuationRecord::not_needed();
6532            let mut final_seat = seat.clone();
6533            match out {
6534                AgentOutcome::Ok(o) => {
6535                    fix.duration_ms = o.duration_ms;
6536                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6537                    // A parsed report standing next to a command this same
6538                    // reply's own CLI never confirmed the exit status of is
6539                    // not a resolved answer — the identical `CommandEvidence`
6540                    // `state.jobs` renders, read here instead of only on
6541                    // display, per the completion judgment and the shown
6542                    // record needing to agree.
6543                    let incomplete_reason = match &parsed {
6544                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6545                            "the reply parsed, but it reported a command whose own CLI never \
6546                             confirmed an exit status"
6547                                .to_owned(),
6548                        ),
6549                        Ok(_) => None,
6550                        Err(e) => Some(e.to_string()),
6551                    };
6552                    match incomplete_reason {
6553                        None => {
6554                            let report = parsed.expect("checked Ok above");
6555                            fix.addressed = report.addressed;
6556                            fix.rejected = report.rejected;
6557                            fix.notes =
6558                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6559                        }
6560                        Some(reason) => {
6561                            let (resumed_seat, resolved, failure, cont) = self
6562                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6563                                .await;
6564                            fix.duration_ms += cont.cumulative_wait_ms;
6565                            continuation = cont;
6566                            final_seat = resumed_seat;
6567                            match resolved {
6568                                Some(report) => {
6569                                    fix.addressed = report.addressed;
6570                                    fix.rejected = report.rejected;
6571                                    fix.notes = blind::sanitize_prose(
6572                                        &report.notes,
6573                                        &self.state.config.blind,
6574                                    );
6575                                }
6576                                None => fix.failed = failure,
6577                            }
6578                        }
6579                    }
6580                }
6581                // The CLI's raw error JSON is not a fix report to parse.
6582                AgentOutcome::Dropped(o) => {
6583                    fix.duration_ms = o.duration_ms;
6584                    let why = o
6585                        .dropped
6586                        .as_ref()
6587                        .map(|d| d.why.as_str())
6588                        .unwrap_or("the CLI ended the stream without delivering its answer");
6589                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6590                }
6591                AgentOutcome::Quota(o) => {
6592                    self.state.quota.push(QuotaLoss {
6593                        seat: final_seat.key.clone(),
6594                        node: "fix".to_owned(),
6595                        at: Timestamp::now(),
6596                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6597                    });
6598                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6599                }
6600                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6601            }
6602            fix.continuation = Some(continuation);
6603            self.state.seats.insert(final_seat.key.clone(), final_seat);
6604            let coordination = self.coordinate(&winner.worktree).await?;
6605            if let Ok(r) = git::rescue_commit(
6606                &winner.worktree,
6607                &format!("magi: review round {round} fixes (uncommitted work)"),
6608            )
6609            .await
6610            {
6611                self.state.note_withheld("fix", &r.withheld);
6612            }
6613            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6614            fix.committed = after != before;
6615            // Judged by what `git` says moved against base, never by the
6616            // fixer's own `addressed`/`rejected` count — see
6617            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6618            // `patch` snapshot above: swallowing this error would default
6619            // `diff_after` to empty, which almost always differs from a
6620            // non-empty `patch` and reads as "progressed" — exactly backwards
6621            // for a `git` failure the stagnation check cannot see through.
6622            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6623            drop(coordination);
6624            let progressed = diff_after != patch;
6625            let commit_note = if fix.committed {
6626                "committed"
6627            } else {
6628                "NO new commit"
6629            };
6630            let tree_note = if progressed {
6631                "changed vs base"
6632            } else {
6633                "unchanged vs base"
6634            };
6635            self.state.event(
6636                "fix",
6637                match &fix.failed {
6638                    // Distinct on purpose from "0 addressed, 0 rejected": the
6639                    // fixer's own diff still landed (blocking counts do keep
6640                    // falling round over round), only its adoption report did
6641                    // not come back, so this must never read like every
6642                    // finding was reviewed and declined.
6643                    Some(reason) => {
6644                        format!(
6645                            "round {round}: fixer's adoption report was lost ({reason}); \
6646                             {commit_note}, tree {tree_note}"
6647                        )
6648                    }
6649                    None => format!(
6650                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6651                         {tree_note}{}",
6652                        fix.addressed.len(),
6653                        fix.rejected.len(),
6654                        if continuation.outcome == ContinuationOutcome::Resumed {
6655                            format!(
6656                                " (adoption report recovered after {} continuation(s))",
6657                                continuation.attempts
6658                            )
6659                        } else {
6660                            String::new()
6661                        },
6662                    ),
6663                },
6664            );
6665            round_record.fix = Some(fix);
6666            round_record.progressed = progressed;
6667            self.state.reviews.push(round_record);
6668            self.state.save()?;
6669
6670            // The fixer's own report never came back this round, even after
6671            // `continue_fix_report`'s own budget was spent on it — not an
6672            // ordinary "no report" (dropped stream, quota, plain failure),
6673            // which already reads that way and is left to the existing round
6674            // budget. Stopping here, rather than opening another round, is
6675            // what keeps a next reviewer/fixer wave from ever being
6676            // dispatched onto `winner.worktree` while whatever the seat's
6677            // last call may still have running there is unaccounted for: no
6678            // process liveness check exists (and none is being added — see
6679            // AGENTS.md/this task's own scope), so the only way to honour
6680            // "nothing starts before a valid report returns" is to not start
6681            // anything further on this worktree from this run at all.
6682            if matches!(
6683                continuation.outcome,
6684                ContinuationOutcome::Exhausted
6685                    | ContinuationOutcome::QuotaLost
6686                    | ContinuationOutcome::NoSession
6687            ) {
6688                return self
6689                    .stop_reviewing(
6690                        "the fixer's adoption report never came back, even after resuming its \
6691                         own seat; refusing to start another round against the same worktree \
6692                         while that is unresolved",
6693                        &shell,
6694                        &winner.worktree,
6695                    )
6696                    .await;
6697            }
6698
6699            let streak = self
6700                .state
6701                .reviews
6702                .iter()
6703                .rev()
6704                .take_while(|r| !r.progressed)
6705                .count();
6706            if streak >= STAGNANT_LIMIT {
6707                return self
6708                    .stop_reviewing(
6709                        &format!(
6710                            "the tree has not moved against base for {streak} round(s) in a row"
6711                        ),
6712                        &shell,
6713                        &winner.worktree,
6714                    )
6715                    .await;
6716            }
6717        }
6718        Ok(())
6719    }
6720
6721    /// Decide, from the last recorded round's own verification, whether
6722    /// stopping the review loop is a hand-off or a genuine block.
6723    ///
6724    /// Called once the loop has given up trying — the round budget is spent,
6725    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6726    /// findings still open, never while a round is still clean or the
6727    /// incomplete-panel case handled inline above. Gate and e2e are facts
6728    /// about the tree; a lingering review finding is an opinion, and this
6729    /// workload's own `magi stats` puts reviewer precision low enough
6730    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6731    /// must not by itself stand between a green, verified change and the
6732    /// human who decides what to do with it. A red e2e is not an opinion, so
6733    /// that case still blocks, with the failing command and a tail of its
6734    /// output recorded here rather than left in `run.json` for someone to go
6735    /// find.
6736    ///
6737    /// A round that deferred its own e2e (see [`Config::graph`]'s
6738    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6739    /// only because nothing ran, and treating an empty list as a passing one
6740    /// here is exactly the "deferred painted green" bug this function exists
6741    /// to not have. When the last round's own verification never resolved —
6742    /// deferred on purpose, or a real attempt the shared build cache blocked
6743    /// — this makes (or retries) the real run, on the actual worktree this
6744    /// loop is about to stop touching, before deciding anything. A
6745    /// resource-blocked attempt is likewise never read as either green or
6746    /// red: it is evidence about the machine, not the patch (see
6747    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6748    /// blocked cache leaves this call without deciding rather than guessing
6749    /// — the caller retries on a later reentry.
6750    /// Record, once, that the review loop handed off over a blocking finding
6751    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6752    /// `land` asks the owner even with `land_approval` off. Called from every
6753    /// path that concludes `Gating`; a reentry keeps the first record.
6754    fn record_contested_handoff(&mut self) {
6755        if self.state.contested_handoff.is_some() {
6756            return;
6757        }
6758        let Some(contested) = self
6759            .state
6760            .reviews
6761            .last()
6762            .and_then(ReviewRound::contested_handoff)
6763        else {
6764            return;
6765        };
6766        self.state.event(
6767            "review",
6768            format!(
6769                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6770                 wait for the owner's approval",
6771                contested.findings.len(),
6772                contested.rejecters.len()
6773            ),
6774        );
6775        self.state.contested_handoff = Some(contested);
6776    }
6777
6778    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6779        let round_idx = self.state.reviews.len() - 1;
6780        // A deferred round and a resource-blocked one are the same shape
6781        // here: neither has a real result yet, and both get one more
6782        // attempt. Read off `e2e_status` — the single source for this —
6783        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6784        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6785        // still retries instead of being read as a settled result the
6786        // instant it stops being empty.
6787        let needs_catchup_run = matches!(
6788            self.state.reviews[round_idx].e2e_status(),
6789            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6790        );
6791        if needs_catchup_run {
6792            let round = self.state.reviews[round_idx].round;
6793            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6794            let commands = self.state.config.verify.e2e.clone();
6795            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6796            let cache_dir = self.state.config.cache_dir();
6797            let context = format!(
6798                "round {round}: verification unresolved, catching up before the final decision"
6799            );
6800            let wait = self.repository_wait();
6801            let (outcomes, verify_retried) = with_cache_lease(
6802                &mut self.state,
6803                cache_dir.as_deref(),
6804                "e2e",
6805                "e2e",
6806                worktree,
6807                &attempted_head,
6808                timeout,
6809                &context,
6810                &wait,
6811                |state, budget| {
6812                    let shell = shell.to_vec();
6813                    let commands = commands.clone();
6814                    let context = context.clone();
6815                    async move {
6816                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6817                            .await
6818                    }
6819                },
6820            )
6821            .await?;
6822            let last = &mut self.state.reviews[round_idx];
6823            last.e2e = outcomes;
6824            last.verify_retried = verify_retried;
6825            // Always the commit and time this attempt actually targeted,
6826            // whether or not it happens to equal the reviewed `head` and
6827            // whether or not a command finished — see
6828            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6829            // attempt is recorded too, so a later reader sees "attempted
6830            // again at T2" rather than silence.
6831            last.verified_head = Some(attempted_head);
6832            last.verified_at = Some(Timestamp::now());
6833            if verify_inconclusive(&last.e2e) {
6834                // Still not a real result: `e2e_deferred` is left exactly
6835                // as it was, so `needs_catchup_run` above reads
6836                // `ResourceBlocked` (via `e2e_status`, which checks
6837                // `resource_blocked` before `e2e_deferred`) and retries
6838                // again on the next reentry, rather than recording
6839                // contention as a red e2e and blocking the run on it.
6840                self.state.save()?;
6841                return Ok(());
6842            }
6843            last.e2e_deferred = false;
6844        }
6845        let last = &self.state.reviews[round_idx];
6846        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6847
6848        match last.e2e_status() {
6849            E2eStatus::Failed => {
6850                let red: Vec<String> = last
6851                    .e2e
6852                    .iter()
6853                    .filter(|o| !o.ok())
6854                    .map(|o| {
6855                        format!(
6856                            "`{}` -> {:?}\n{}",
6857                            o.command,
6858                            o.code,
6859                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6860                        )
6861                    })
6862                    .collect();
6863                self.state
6864                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6865                self.state.status = RunStatus::Blocked;
6866            }
6867            // `needs_catchup_run` above already retried once this call; if
6868            // it is still blocked, this is magi's own admission it could
6869            // not get a command to run, never a verdict on the patch — the
6870            // run is left exactly where a later reentry can retry again.
6871            E2eStatus::ResourceBlocked => {
6872                self.state.event(
6873                    "review",
6874                    format!(
6875                        "{why}; e2e could not run (shared build cache unavailable); not \
6876                         deciding yet"
6877                    ),
6878                );
6879            }
6880            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6881                self.state.event(
6882                    "review",
6883                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6884                );
6885                self.record_contested_handoff();
6886                self.state.status = RunStatus::Gating;
6887            }
6888        }
6889        self.state.save()?;
6890        Ok(())
6891    }
6892
6893    // ----------------------------------------------------------------- gate
6894
6895    async fn gate(&mut self) -> Result<()> {
6896        // Judged by the review record itself, not by `status`: a solo
6897        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6898        // reentry (see `judge`), and trusting it here is exactly how a run
6899        // that exhausted its review budget got gated and merged a second
6900        // time around. `review_conclusion` recomputes the review loop's own
6901        // verdict from the round records themselves — `Gating` for a clean
6902        // round or a hand-off (see `stop_reviewing`), anything else means the
6903        // loop is still going or genuinely blocked.
6904        // A base the winner could not be replayed onto is a decision, not a
6905        // round: there is no landing tree to gate. Read as its own record for
6906        // the same reason the review verdict is.
6907        if self.state.status == RunStatus::Failed
6908            || self
6909                .state
6910                .base_sync
6911                .as_ref()
6912                .is_some_and(|s| s.conflict.is_some())
6913            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6914                != Some(RunStatus::Gating)
6915        {
6916            return Ok(());
6917        }
6918        if self.state.gate_ran {
6919            // `review_loop` derives its conclusion from the clean review
6920            // record on every reentry and therefore puts a completed run back
6921            // in `Gating`. A recorded gate is a stronger, terminal fact:
6922            // retain its original command output (or lack of any, for a repo
6923            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6924            // doc) and restore `Blocked` on a real failure rather than
6925            // pretending the command is still running or running it a second
6926            // time. `gate_ran == false` remains the only shape — unattempted,
6927            // or a resource-blocked retry — that may still need to execute a
6928            // command.
6929            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6930                self.state.status = RunStatus::Blocked;
6931                self.state.save()?;
6932            }
6933            return Ok(());
6934        }
6935        let Some(winner) = self.state.winner().cloned() else {
6936            return Ok(());
6937        };
6938        self.state.status = RunStatus::Gating;
6939        let mut outcomes = self.run_gate(&winner).await?;
6940        loop {
6941            // A resource-blocked outcome means the gate command never actually
6942            // ran - the shared build cache could not be acquired or confirmed
6943            // fresh in time - which is evidence about the machine, not about
6944            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6945            // Recording it as a red gate would mark a run `Blocked` on nothing
6946            // but contention magi has already logged; leaving `self.state.gate`
6947            // empty and `self.state.gate_ran` false instead keeps the shape
6948            // this function already treats as "still needs to run" (see the
6949            // early-return above), so the next call retries the command
6950            // rather than concluding anything.
6951            if verify_inconclusive(&outcomes) {
6952                self.state.save()?;
6953                return Ok(());
6954            }
6955            if outcomes.iter().all(CommandOutcome::ok) {
6956                break;
6957            }
6958            match self.gate_fix_round(&winner, &outcomes).await? {
6959                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6960                GateFix::Stop => break,
6961                GateFix::Defer => {
6962                    self.state.save()?;
6963                    return Ok(());
6964                }
6965            }
6966        }
6967        let passed = outcomes.iter().all(CommandOutcome::ok);
6968        self.state.gate = outcomes;
6969        self.state.gate_ran = true;
6970        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6971        if !passed {
6972            self.state.status = RunStatus::Blocked;
6973            let spent = self.state.gate_fixes.len();
6974            self.state.event(
6975                "gate",
6976                if spent == 0 {
6977                    "gate failed; not merging".to_owned()
6978                } else {
6979                    format!("gate failed after {spent} gate-fix round(s); not merging")
6980                },
6981            );
6982        }
6983        self.state.save()?;
6984        Ok(())
6985    }
6986
6987    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
6988    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
6989    /// after review is clean and never on a candidate awaiting judging.
6990    ///
6991    /// Never fails the run: a non-zero exit or timeout is a warning and a
6992    /// recorded outcome, and the gate remains the single arbiter. Nothing
6993    /// configured means nothing happens - no event, no commit. `commit_all`
6994    /// commits any leftover change under the neutral identity and returns
6995    /// `false` when the tree is clean, so no empty commit is ever made.
6996    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6997        let commands = state.config.verify.pre_gate.clone();
6998        if commands.is_empty() {
6999            return;
7000        }
7001        let shell = state.config.shell();
7002        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7003        let (outcomes, _) = run_commands(
7004            state,
7005            "pre_gate",
7006            "pre_gate",
7007            0,
7008            &shell,
7009            &commands,
7010            &winner.worktree,
7011            timeout,
7012        )
7013        .await;
7014        for o in &outcomes {
7015            if !o.ok() {
7016                tracing::warn!(
7017                    "pre_gate `{}` failed ({:?}); the gate decides",
7018                    o.command,
7019                    o.code
7020                );
7021            }
7022            state.event(
7023                "pre_gate",
7024                format!(
7025                    "`{}` -> {}",
7026                    o.command,
7027                    if o.ok() {
7028                        "pass".to_owned()
7029                    } else {
7030                        format!(
7031                            "FAIL ({:?})\n{}",
7032                            o.code,
7033                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7034                        )
7035                    }
7036                ),
7037            );
7038        }
7039        state.pre_gate = outcomes;
7040        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7041            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7042                Ok(head) => {
7043                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7044                    state.pre_gate_commit = Some(head);
7045                }
7046                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7047            },
7048            Ok(false) => {}
7049            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7050        }
7051        if let Err(e) = state.save() {
7052            tracing::warn!("could not persist the pre_gate record: {e:#}");
7053        }
7054    }
7055
7056    /// Run `verify.gate` once against the winner's current tree, logging one
7057    /// event per command. Empty when nothing is configured.
7058    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7059        let wait = self.repository_wait();
7060        run_landing_gate(&mut self.state, winner, &wait).await
7061    }
7062
7063    /// One bounded fix round for a failing gate.
7064    ///
7065    /// The fixer is told the failure came from the gate itself, not from a
7066    /// reviewer, and is shown the failed commands, their exit codes and a tail
7067    /// of their output - whatever `[verify].gate` holds, nothing here knows
7068    /// what those commands run. Only a normal non-zero exit that printed
7069    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7070    /// command or a full disk says nothing about the code, and a fixer sent
7071    /// after it can only appease the machine. The round is judged by what git
7072    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7073    /// again before the gate does, so a fix cannot trade a green gate for a
7074    /// red e2e unnoticed.
7075    async fn gate_fix_round(
7076        &mut self,
7077        winner: &Candidate,
7078        outcomes: &[CommandOutcome],
7079    ) -> Result<GateFix> {
7080        let cap = self.state.config.graph.gate_fix_rounds;
7081        let spent = self.state.gate_fixes.len();
7082        if spent >= cap {
7083            if cap > 0 {
7084                self.state.event(
7085                    "gate",
7086                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7087                );
7088            }
7089            return Ok(GateFix::Stop);
7090        }
7091        if !gate_fixable(outcomes) {
7092            self.state.event(
7093                "gate",
7094                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7095                 command or similar); not spending a fix round on it",
7096            );
7097            return Ok(GateFix::Stop);
7098        }
7099        let min_free = self.state.config.disk.min_free_bytes;
7100        if min_free > 0 {
7101            match crate::disk::free_bytes(&winner.worktree) {
7102                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7103                Ok(free) => {
7104                    self.state.event(
7105                        "gate",
7106                        format!(
7107                            "only {free} bytes free ({min_free} required by `[disk] \
7108                             min_free_bytes`); not spending a fix round on a failure the disk \
7109                             may explain"
7110                        ),
7111                    );
7112                    return Ok(GateFix::Stop);
7113                }
7114                Err(e) => {
7115                    self.state.event(
7116                        "gate",
7117                        format!("free disk space could not be measured ({e:#}); no fix round"),
7118                    );
7119                    return Ok(GateFix::Stop);
7120                }
7121            }
7122        }
7123
7124        let attempt = spent + 1;
7125        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7126        let base = self.landing_base();
7127        let fix_prompt = prompt::gate_fix_in(
7128            &self.state.instruction,
7129            &failed,
7130            attempt,
7131            cap,
7132            &self.state.config.graph.language,
7133            &self.state.config.graph.github_language,
7134        );
7135        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7136        let sessions = self.state.config.graph.sessions;
7137        let artifacts = self.merge_repair_artifacts();
7138        self.state.event(
7139            "gate",
7140            format!("gate failed; gate-fix round {attempt} of {cap}"),
7141        );
7142        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7143        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7144        // Persist before invoking the fixer: it may commit and then be
7145        // interrupted before its result (or the following e2e) is saved.
7146        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7147            head: before.clone(),
7148            passed: false,
7149            outcomes: Vec::new(),
7150        });
7151        self.state.save()?;
7152        let (_, seat, out) = self
7153            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7154                prompt: fix_prompt.clone(),
7155                spec,
7156                seat,
7157                cwd: winner.worktree.clone(),
7158                timeout,
7159                allow_write: true,
7160                sessions,
7161                artifacts: artifacts.clone(),
7162                stem: format!("gate-fix-{attempt}"),
7163                handover: None,
7164            })
7165            .await;
7166        // The fixer was never asked: no round is spent (nothing is recorded)
7167        // and a later reentry asks the same agent again.
7168        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7169            return Ok(GateFix::Defer);
7170        }
7171        let mut record = GateFixRecord {
7172            agent: seat.agent.clone(),
7173            failed,
7174            notes: String::new(),
7175            committed: false,
7176            error: None,
7177        };
7178        match out {
7179            AgentOutcome::Ok(o) => {
7180                // A missing report is not a failed fix: the round is judged
7181                // by the tree below, and the report only carries prose.
7182                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7183                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7184                }
7185            }
7186            AgentOutcome::Dropped(_) => {
7187                record.error = Some("the CLI dropped the stream".to_owned());
7188            }
7189            AgentOutcome::Quota(o) => {
7190                self.state.quota.push(QuotaLoss {
7191                    seat: seat.key.clone(),
7192                    node: "gate-fix".to_owned(),
7193                    at: Timestamp::now(),
7194                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7195                });
7196                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7197            }
7198            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7199        }
7200        self.state.seats.insert(seat.key.clone(), seat);
7201        let coordination = self.coordinate(&winner.worktree).await?;
7202        if let Ok(r) = git::rescue_commit(
7203            &winner.worktree,
7204            &format!("magi: gate fix {attempt} (uncommitted work)"),
7205        )
7206        .await
7207        {
7208            self.state.note_withheld("gate-fix", &r.withheld);
7209        }
7210        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7211        record.committed = after != before;
7212        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7213        let note = record.error.clone();
7214        self.state.gate_fixes.push(record);
7215        self.state.save()?;
7216        if !changed {
7217            self.state.event(
7218                "gate",
7219                match note {
7220                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7221                    None => format!("gate-fix round {attempt}: the tree did not change"),
7222                },
7223            );
7224            return Ok(GateFix::Stop);
7225        }
7226        self.state.event(
7227            "gate",
7228            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7229        );
7230
7231        drop(coordination);
7232        let wait = self.repository_wait();
7233        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7234    }
7235
7236    // ---------------------------------------------------------------- merge
7237
7238    /// One read-only rewrite by the summary's author; a text the gate still
7239    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7240    /// applies on their word or on silence. `None` means the run is parked on
7241    /// that question and nothing may be pushed or opened yet.
7242    async fn guarded_pr_message(
7243        &mut self,
7244        winner: &Candidate,
7245        facts: Option<&BranchFacts>,
7246        posting: bool,
7247    ) -> Result<Option<PrMessage>> {
7248        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7249        let head = match final_head.as_deref() {
7250            Some(h) => crate::pr_outcome::FinalHead::At(h),
7251            None => crate::pr_outcome::FinalHead::Unknown,
7252        };
7253        // Only a pull request that is about to be posted pays for a summary.
7254        let summarized = if posting {
7255            Some(self.summarize_change(winner, final_head.as_deref()).await)
7256        } else {
7257            None
7258        };
7259        // A busy shared resource is not a permanent cause: publishing an
7260        // "unavailable" description would stick (a later head move never
7261        // edits the body). Stop before anything is pushed or opened; the
7262        // run is `Blocked` and a resume asks the summary again.
7263        if self.state.resource_block.is_some() {
7264            self.state.event(
7265                "pr-summary",
7266                "a shared resource was busy; nothing was published, resume to summarize again",
7267            );
7268            return Ok(None);
7269        }
7270        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7271            anyhow::bail!(
7272                "the final commit moved during public description generation; retry before publishing"
7273            );
7274        }
7275        let note = match (&summarized, &self.state.change_summary) {
7276            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7277            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7278            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7279            (None, _) => ChangeNote::NotAsked,
7280        };
7281        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7282        if !posting {
7283            let identity = crate::scrub::Identity::current();
7284            return Ok(Some(PrMessage {
7285                title: crate::scrub::scrub(&pr.title, &identity),
7286                body: crate::scrub::scrub(&pr.body, &identity),
7287            }));
7288        }
7289        // This very text was already rejected and asked about: no second
7290        // judge call, rewrite or question, just carry the record on.
7291        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7292        if let Some(known) = self
7293            .state
7294            .github_text
7295            .clone()
7296            .filter(|g| g.fingerprint == raw_fp)
7297        {
7298            return self.settle_withheld(winner, &pr, known).await;
7299        }
7300        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7301        let mut violations =
7302            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7303        let identity = crate::scrub::Identity::current();
7304        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7305            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7306            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7307        {
7308            violations.push(crate::github_text::Violation::SensitiveData);
7309        }
7310        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7311        // only a language violation starts one; sensitive data alone goes
7312        // straight to the owner, with the redacted text as the fallback.
7313        let has_language = violations
7314            .iter()
7315            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7316        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7317            self.state.event(
7318                "github-text",
7319                if has_language {
7320                    format!("description rejected: {violations:?}; requesting one rewrite")
7321                } else {
7322                    format!("description rejected: {violations:?}; asking the owner")
7323                },
7324            );
7325            let mut rewritten = false;
7326            let mut rejected_rewrite = None;
7327            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7328            {
7329                let output = text;
7330                if let Some(title) = summary_title(&output) {
7331                    // Judge the redacted candidate that will actually be posted.
7332                    // Sensitive data alone must not discard a useful rewrite.
7333                    let raw_body = summary_without_title(&output);
7334                    let clean_title = crate::scrub::scrub(&title, &identity);
7335                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7336                    let redacted = clean_title != title || body != raw_body;
7337                    let title = clean_title;
7338                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7339                    if !complete {
7340                        self.state.event(
7341                            "github-text",
7342                            "description rewrite omitted required findings",
7343                        );
7344                    }
7345                    if !body.trim().is_empty() {
7346                        body.push_str("\n\n");
7347                        body.push_str(&public_pr_evidence(&self.state, head));
7348                    }
7349                    let marker = format!("magi:run/{}", self.state.id);
7350                    if !body.trim().is_empty() && !body.contains(&marker) {
7351                        body.push_str(&format!("\n\n{marker}"));
7352                    }
7353                    let rewrite = PrMessage {
7354                        title: title.clone(),
7355                        body: body.clone(),
7356                    };
7357                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7358                    let mut failed = crate::github_text::check_for(
7359                        &self.state.config.graph,
7360                        &title,
7361                        &body,
7362                        redecision,
7363                    );
7364                    if (!crate::github_text::shareable(&title)
7365                        || !crate::github_text::shareable(&body))
7366                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7367                    {
7368                        failed.push(crate::github_text::Violation::SensitiveData);
7369                    }
7370                    if complete
7371                        && !title.trim().is_empty()
7372                        && !body.trim().is_empty()
7373                        && !failed.is_empty()
7374                    {
7375                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7376                            title: title.clone(),
7377                            body: body.clone(),
7378                            failed: failed
7379                                .iter()
7380                                .map(|v| crate::github_text::category(*v).to_owned())
7381                                .collect(),
7382                        });
7383                    }
7384                    if complete
7385                        && !title.trim().is_empty()
7386                        && !raw_body.trim().is_empty()
7387                        && failed.is_empty()
7388                    {
7389                        decision = redecision;
7390                        pr = PrMessage { title, body };
7391                        rewritten = true;
7392                        self.state
7393                            .event("github-text", "description rewrite passed");
7394                        if redacted {
7395                            self.state.event(
7396                                "github-text",
7397                                "sensitive data removed from description rewrite",
7398                            );
7399                        }
7400                    }
7401                }
7402            }
7403            if !rewritten {
7404                self.state.event(
7405                    "github-text",
7406                    "description rewrite unavailable or rejected; asking the owner",
7407                );
7408                if let Some(w) =
7409                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7410                {
7411                    // A different text than the one asked about before: that
7412                    // question no longer describes anything.
7413                    self.retire_github_text_question("the withheld text changed");
7414                    let record = crate::run::GithubTextAsk {
7415                        rewrite: rejected_rewrite,
7416                        fingerprint: raw_fp,
7417                        title: w.title,
7418                        body: w.body,
7419                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7420                        question: None,
7421                        asks: 0,
7422                        resolved: false,
7423                        chosen_title: None,
7424                        choice: None,
7425                    };
7426                    return self.settle_withheld(winner, &pr, record).await;
7427                }
7428            }
7429        }
7430        let (title, body) =
7431            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7432        Ok(Some(PrMessage { title, body }))
7433    }
7434
7435    /// Describe the whole change from its final diff in either run mode.
7436    ///
7437    /// The summary is tied to the final commit and to the merge base of the
7438    /// resolved remote base; a record for any other pair is dropped and the
7439    /// agent asked again, so a description never outlives the diff it was
7440    /// written from. The seat is read-only and the prompt tells it to read the
7441    /// diff and the code around it. `[roles] synthesizer` selects the agents
7442    /// (unset: the default pick), each asked at most once, advancing on an
7443    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7444    /// carries the reason it is unavailable; nothing is invented then.
7445    async fn summarize_change(
7446        &mut self,
7447        winner: &Candidate,
7448        head: Option<&str>,
7449    ) -> std::result::Result<(), String> {
7450        let Some(head) = head else {
7451            return Err("the final commit could not be read".to_owned());
7452        };
7453        let remote = self.state.config.merge.remote.clone();
7454        let tracking = format!("{remote}/{}", self.state.base_branch);
7455        if !git::rev_exists(&winner.worktree, &tracking).await {
7456            return Err(format!("the remote base `{tracking}` could not be read"));
7457        }
7458        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7459            Ok(b) if !b.is_empty() => b,
7460            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7461        };
7462        let evidence = crate::pr_outcome::summary_evidence(
7463            &self.state,
7464            crate::pr_outcome::FinalHead::At(head),
7465        );
7466        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7467        if self
7468            .state
7469            .change_summary
7470            .as_ref()
7471            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7472        {
7473            return Ok(());
7474        }
7475        if self.state.change_summary.take().is_some() {
7476            self.state.event(
7477                "pr-summary",
7478                "the final diff or evidence changed; the earlier change summary was discarded",
7479            );
7480        }
7481        // Save the source snapshot even if no generation agent is available.
7482        if let Err(e) = write_artifact(
7483            &self.state,
7484            &format!("public-description-{fingerprint}.md"),
7485            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7486        ) {
7487            tracing::warn!("could not save public description evidence: {e:#}");
7488        }
7489        let chain = match agent::pick_chain(
7490            &self.state.config.agents,
7491            self.state.config.roles.synthesizer.as_ref(),
7492            &agent::installed,
7493            "synthesizer",
7494        ) {
7495            Ok(chain) => chain,
7496            Err(e) => {
7497                tracing::warn!("no agent for the change summary: {e:#}");
7498                self.state.event("pr-summary", "no summary agent available");
7499                return Err("no agent was available to summarize it".to_owned());
7500            }
7501        };
7502        let subjects = git::subjects(&winner.worktree, &base, head)
7503            .await
7504            .unwrap_or_default();
7505        let stat = git::diff_stat(&winner.worktree, &base, head)
7506            .await
7507            .unwrap_or_default();
7508        let stat: String = stat.chars().take(6_000).collect();
7509        let mut prompt = crate::prompt::change_summary_in(
7510            &base,
7511            head,
7512            &subjects,
7513            stat.trim(),
7514            &self.state.config.graph.github_language,
7515        );
7516        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7517        let sessions = self.state.config.graph.sessions;
7518        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7519        let prompts = self.state.config.prompts.clone();
7520        let cache = self.state.config.cache_dir();
7521        let run = self.state.id.clone();
7522        for (i, spec) in chain.into_iter().enumerate() {
7523            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7524            let job = SeatJob {
7525                spec: spec.clone(),
7526                seat,
7527                cwd: winner.worktree.clone(),
7528                prompt: prompt.clone(),
7529                timeout,
7530                allow_write: false,
7531                sessions,
7532                artifacts: agent::artifacts_dir(&self.state.dir()),
7533                stem: if i == 0 {
7534                    "change-summary".to_owned()
7535                } else {
7536                    format!("change-summary-{}", spec.id)
7537                },
7538                handover: None,
7539            };
7540            let ctx = WaveCtx {
7541                run: &run,
7542                node: "pr-summary",
7543                prompts: &prompts,
7544                cache: cache.as_deref(),
7545                round: None,
7546                carry_seats: false,
7547                pauses: Some((&self.pause, &self.interrupt)),
7548            };
7549            let (seat, outcome) =
7550                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7551            self.state.seats.insert(seat.key.clone(), seat);
7552            let text = match outcome {
7553                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7554                AgentOutcome::ResourceBlocked(why) => {
7555                    // No agent was asked: the run stops resumably before
7556                    // anything is published (see `guarded_pr_message`).
7557                    if self.state.resource_block.is_none() {
7558                        self.state.resource_block = Some(crate::run::ResourceBlock {
7559                            node: "pr-summary".to_owned(),
7560                            kind: "shared resource".to_owned(),
7561                            detail: why,
7562                            at: Timestamp::now(),
7563                        });
7564                    }
7565                    return Err("a shared resource was busy".to_owned());
7566                }
7567                _ => continue,
7568            };
7569            let blind = &self.state.config.blind;
7570            let Some(title) = summary_title(&text) else {
7571                continue;
7572            };
7573            let body: String = summary_without_title(&text)
7574                .lines()
7575                .filter(|l| !l.contains("magi:run/"))
7576                .collect::<Vec<_>>()
7577                .join("\n");
7578            let body = blind::sanitize_prose(body.trim(), blind);
7579            let title = blind::sanitize_prose(&title, blind);
7580            if body.trim().is_empty() || title.trim().is_empty() {
7581                continue;
7582            }
7583            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7584                continue;
7585            }
7586            // The branch must still be the one the agent was shown.
7587            if git::rev_parse(&winner.worktree, "HEAD")
7588                .await
7589                .ok()
7590                .as_deref()
7591                != Some(head)
7592            {
7593                return Err("the branch moved while the summary was written".to_owned());
7594            }
7595            if git::merge_base(&winner.worktree, &tracking, head)
7596                .await
7597                .ok()
7598                .as_deref()
7599                != Some(base.as_str())
7600                || crate::pr_outcome::summary_evidence(
7601                    &self.state,
7602                    crate::pr_outcome::FinalHead::At(head),
7603                ) != evidence
7604            {
7605                return Err("the summary evidence changed while it was written".to_owned());
7606            }
7607            self.state.change_summary = Some(crate::run::ChangeSummary {
7608                head: head.to_owned(),
7609                base,
7610                evidence: fingerprint,
7611                title,
7612                body,
7613            });
7614            self.state.event(
7615                "pr-summary",
7616                format!("change summary written by {} for the final diff", spec.id),
7617            );
7618            if let Err(e) = self.state.save() {
7619                tracing::warn!("could not save the change summary: {e:#}");
7620            }
7621            return Ok(());
7622        }
7623        self.state
7624            .event("pr-summary", "no agent produced a change summary");
7625        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7626    }
7627
7628    /// One read-only rewrite request for a withheld description. The
7629    /// implementer's own conversation is reused only when it exists and can be
7630    /// resumed; otherwise the `[roles] language_judge` chain (or the default
7631    /// pick) writes it in a separate conversation. The prompt carries the whole
7632    /// text, so no agent needs any memory. Each agent is asked at most once and
7633    /// the chain advances on an error, quota, unusable answer or a reply with
7634    /// no `TITLE:` line. Nothing is recorded as a handover.
7635    async fn rewrite_pr_text(
7636        &mut self,
7637        winner: &Candidate,
7638        pr: &PrMessage,
7639        violations: &[crate::github_text::Violation],
7640    ) -> Option<String> {
7641        let prompt = format!(
7642            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7643            pr.title, pr.body
7644        );
7645        let sessions = self.state.config.graph.sessions;
7646        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7647        // (spec, seat, resumed)
7648        let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7649        let key = format!("impl-{}", winner.label);
7650        if let Some(spec) = self
7651            .state
7652            .config
7653            .agents
7654            .iter()
7655            .find(|a| a.id == winner.agent)
7656            .cloned()
7657            && let Some(existing) = self.state.seats.get(&key)
7658            && existing.agent == spec.id
7659            && agent::has_session(spec.kind, existing, sessions)
7660        {
7661            attempts.push((spec, existing.clone(), true));
7662        } else {
7663            let chain = agent::pick_chain(
7664                &self.state.config.agents,
7665                self.state.config.roles.language_judge.as_ref(),
7666                &agent::installed,
7667                "language_judge",
7668            );
7669            match chain {
7670                Ok(chain) => {
7671                    for spec in chain {
7672                        let seat = crate::fixer::seat_for(
7673                            &mut self.state,
7674                            "github-text-rewrite",
7675                            &spec.id,
7676                        );
7677                        attempts.push((spec, seat, false));
7678                    }
7679                }
7680                Err(e) => {
7681                    tracing::warn!("no agent for the description rewrite: {e:#}");
7682                    self.state
7683                        .event("github-text", "no rewrite agent available");
7684                    return None;
7685                }
7686            }
7687        }
7688        let prompts = self.state.config.prompts.clone();
7689        let cache = self.state.config.cache_dir();
7690        let run = self.state.id.clone();
7691        for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7692            let job = SeatJob {
7693                spec: spec.clone(),
7694                seat,
7695                cwd: winner.worktree.clone(),
7696                prompt: prompt.clone(),
7697                timeout: retry_budget(timeout_full, resumed),
7698                allow_write: false,
7699                sessions,
7700                artifacts: agent::artifacts_dir(&self.state.dir()),
7701                stem: if i == 0 {
7702                    "github-text-rewrite".to_owned()
7703                } else {
7704                    format!("github-text-rewrite-{}", spec.id)
7705                },
7706                handover: None,
7707            };
7708            let ctx = WaveCtx {
7709                run: &run,
7710                node: "github-text",
7711                prompts: &prompts,
7712                cache: cache.as_deref(),
7713                round: None,
7714                carry_seats: false,
7715                pauses: Some((&self.pause, &self.interrupt)),
7716            };
7717            let (seat, outcome) =
7718                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7719            self.state.seats.insert(seat.key.clone(), seat);
7720            match outcome {
7721                AgentOutcome::Ok(o)
7722                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7723                {
7724                    return Some(o.text);
7725                }
7726                AgentOutcome::ResourceBlocked(_) => return None,
7727                _ => {}
7728            }
7729        }
7730        None
7731    }
7732
7733    /// Abandon the standing posting-gate question, if any.
7734    fn retire_github_text_question(&mut self, why: &str) {
7735        let Some(id) = self
7736            .state
7737            .github_text
7738            .as_ref()
7739            .and_then(|g| g.question.clone())
7740        else {
7741            return;
7742        };
7743        let _ = ask::Questions::open().update(&id, |q| {
7744            q.abandon(why);
7745            Ok(())
7746        });
7747    }
7748
7749    /// Carry a withheld text to its end: file the question, park while it is
7750    /// open, and on an answer, silence or timeout pick the text to post. The
7751    /// decision is saved before it is used, so it applies once.
7752    async fn settle_withheld(
7753        &mut self,
7754        winner: &Candidate,
7755        pr: &PrMessage,
7756        mut g: crate::run::GithubTextAsk,
7757    ) -> Result<Option<PrMessage>> {
7758        use crate::github_text as gt;
7759        let store = ask::Questions::open();
7760        let timeout = self.state.config.graph.answer_timeout;
7761        if !g.resolved {
7762            let mut standing = g
7763                .question
7764                .as_ref()
7765                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7766            if let Some(q) = standing.as_ref()
7767                && q.status.open()
7768                && gt::expired(q, timeout)
7769            {
7770                let why = format!("no answer within {}s of asking", timeout.max(1));
7771                standing = store
7772                    .update(&q.id, |q| {
7773                        q.abandon(&why);
7774                        Ok(())
7775                    })
7776                    .ok()
7777                    .map(|(q, ())| q);
7778            }
7779            let mut ask_again: Option<Option<Vec<String>>> = None;
7780            match standing {
7781                // Recorded but never written (a stop between the two saves),
7782                // or removed: ask again without spending the retry.
7783                None if g.asks <= 1 => {
7784                    g.asks = 0;
7785                    ask_again = Some(None);
7786                }
7787                None => {
7788                    g.resolved = true;
7789                    g.chosen_title = None;
7790                    g.choice = Some(GithubTextChoice::Fallback);
7791                }
7792                Some(q) if q.status.open() => {
7793                    self.park_on_github_text(&g)?;
7794                    return Ok(None);
7795                }
7796                Some(q) => match gt::read_reply(&q) {
7797                    gt::Reply::Fallback => {
7798                        g.resolved = true;
7799                        g.chosen_title = None;
7800                        g.choice = Some(GithubTextChoice::Fallback);
7801                    }
7802                    gt::Reply::Original => {
7803                        g.resolved = true;
7804                        g.chosen_title = None;
7805                        g.choice = Some(GithubTextChoice::Original);
7806                    }
7807                    gt::Reply::Rewrite => {
7808                        g.resolved = true;
7809                        g.chosen_title = None;
7810                        g.choice = Some(if g.rewrite.is_some() {
7811                            GithubTextChoice::Rewrite
7812                        } else {
7813                            GithubTextChoice::Fallback
7814                        });
7815                    }
7816                    gt::Reply::Title(text) => {
7817                        let first = text
7818                            .lines()
7819                            .map(str::trim)
7820                            .find(|l| !l.is_empty())
7821                            .unwrap_or("");
7822                        let decision = if first.is_empty() {
7823                            None
7824                        } else {
7825                            let candidate = PrMessage {
7826                                title: first.to_owned(),
7827                                body: String::new(),
7828                            };
7829                            self.judge_pr_language(&winner.worktree, &candidate).await
7830                        };
7831                        match gt::vet_title(
7832                            &text,
7833                            decision,
7834                            self.state.config.graph.github_english(),
7835                        ) {
7836                            Ok(title) => {
7837                                g.resolved = true;
7838                                g.chosen_title = Some(title);
7839                                g.choice = Some(GithubTextChoice::MyTitle);
7840                            }
7841                            Err(cats) if g.asks < 2 => {
7842                                ask_again = Some(Some(
7843                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7844                                ))
7845                            }
7846                            Err(_) => {
7847                                g.resolved = true;
7848                                g.chosen_title = None;
7849                                g.choice = Some(GithubTextChoice::Fallback);
7850                            }
7851                        }
7852                    }
7853                },
7854            }
7855            if let Some(rejected) = ask_again {
7856                // A retry (after a rejected replacement title) is still about
7857                // the original text: the same ORIGINAL / FALLBACK comparison.
7858                let w = gt::Withheld {
7859                    title: g.title,
7860                    body: g.body,
7861                    categories: g.categories.clone(),
7862                };
7863                // Named by the original text's fingerprint alone, so a resume
7864                // rewrites the same file and the detail stays identical.
7865                let artifact = match crate::run::write_artifact(
7866                    &self.state,
7867                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7868                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7869                ) {
7870                    Ok(p) => Some(p),
7871                    Err(e) => {
7872                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7873                        None
7874                    }
7875                };
7876                let (detail, comparison) = gt::question_content(
7877                    &w,
7878                    &pr.title,
7879                    &pr.body,
7880                    &self.state.id,
7881                    rejected.as_deref(),
7882                    artifact.as_deref(),
7883                    g.rewrite.as_ref(),
7884                );
7885                let mut q = ask::Question::new(
7886                    self.state.id.clone(),
7887                    gt::ASK_NODE.to_owned(),
7888                    gt::ASK_SEAT.to_owned(),
7889                    gt::question_summary(&self.state.config.graph.language, &w),
7890                    detail,
7891                    gt::question_choices(&w, g.rewrite.as_ref()),
7892                );
7893                q.posting_comparison = Some(comparison);
7894                q.answer_timeout = timeout;
7895                // Recorded before it is written: a stop in between leaves a
7896                // record naming a question the resume then files again.
7897                g.question = Some(q.id.clone());
7898                g.asks += 1;
7899                self.state.github_text = Some(g.clone());
7900                self.state.event(
7901                    "github-text",
7902                    format!("asking the owner about the withheld text ({})", q.short()),
7903                );
7904                self.state.save()?;
7905                store
7906                    .put_executable(&mut q)
7907                    .context("file the posting-gate question")?;
7908                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7909                    tracing::warn!(
7910                        "could not notify about posting-gate question {}: {e:#}",
7911                        q.short()
7912                    );
7913                }
7914                self.park_on_github_text(&g)?;
7915                return Ok(None);
7916            }
7917        }
7918        let resolved = g.chosen_title.clone();
7919        let original = g.choice == Some(GithubTextChoice::Original);
7920        // Field names only: the event log never records the text.
7921        self.state.event(
7922            "github-text",
7923            if original {
7924                format!(
7925                    "posting the original text for: {}",
7926                    match (g.title, g.body) {
7927                        (true, true) => "title, description",
7928                        (true, false) => "title",
7929                        _ => "description",
7930                    }
7931                )
7932            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7933                "posting the saved rewrite title and description".to_owned()
7934            } else if resolved.is_some() {
7935                "posting the owner's replacement title".to_owned()
7936            } else {
7937                "posting the neutral text for the withheld fields".to_owned()
7938            },
7939        );
7940        self.state.github_text = Some(g.clone());
7941        self.state.save()?;
7942        if g.choice == Some(GithubTextChoice::Rewrite)
7943            && let Some(r) = g.rewrite.as_ref()
7944        {
7945            return Ok(Some(PrMessage {
7946                title: r.title.clone(),
7947                body: r.body.clone(),
7948            }));
7949        }
7950        let w = gt::Withheld {
7951            title: g.title,
7952            body: g.body,
7953            categories: g.categories.clone(),
7954        };
7955        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7956        if let Some(t) = resolved {
7957            title = t;
7958        }
7959        // Every withheld field is now neutral or vetted English.
7960        let english = gt::LanguageDecision {
7961            title_english: true,
7962            body_english: true,
7963            body_complete: true,
7964        };
7965        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7966        if original {
7967            // The owner's explicit choice overrides the gate for the fields it
7968            // was made for; the others keep what the gate produced.
7969            if g.title {
7970                title = pr.title.clone();
7971            }
7972            if g.body {
7973                body = pr.body.clone();
7974            }
7975        }
7976        Ok(Some(PrMessage { title, body }))
7977    }
7978
7979    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7980        self.state.github_text = Some(g.clone());
7981        self.state.end_pass_parked(ParkCause::Question);
7982        self.state.parked = true;
7983        self.state.event(
7984            "github-text",
7985            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
7986        );
7987        self.state.save()
7988    }
7989
7990    /// Ask `[roles] language_judge` about this text and record which source
7991    /// decided; the text itself is never recorded.
7992    async fn judge_pr_language(
7993        &mut self,
7994        cwd: &Path,
7995        pr: &PrMessage,
7996    ) -> Option<crate::github_text::LanguageDecision> {
7997        self.state.config.roles.language_judge.as_ref()?;
7998        if !self.state.config.graph.language_gate() {
7999            return None;
8000        }
8001        let decision =
8002            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8003        self.state.event(
8004            "github-text",
8005            if decision.is_some() {
8006                "language decided by the language judge"
8007            } else {
8008                "language judge unavailable; using built-in heuristics"
8009            },
8010        );
8011        decision
8012    }
8013
8014    async fn merge(&mut self) -> Result<()> {
8015        // Same reasoning as `gate`: ask the review and gate records directly
8016        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8017        // skip can rewrite on reentry to something that no longer says
8018        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8019        // so a hand-off (open findings, green verification) reaches merge
8020        // exactly like a genuinely clean round does.
8021        //
8022        // A run resumed mid-`land` never reaches here at all: `execute`
8023        // recognises `RunStatus::Landing` before it even calls `prep`, and
8024        // routes straight to `run_land` instead. That has to happen a level
8025        // up from this function, not with a check in here, because
8026        // `review_loop`'s own status recomputation (see its doc) runs
8027        // *before* `merge` on every reentry and would otherwise overwrite
8028        // the `Landing` marker with `Gating` before this node ever saw it.
8029        if self
8030            .state
8031            .base_sync
8032            .as_ref()
8033            .is_some_and(|s| s.conflict.is_some())
8034            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8035                != Some(RunStatus::Gating)
8036            // `gate_ran == false` is not "passed" - `gate` leaves it false
8037            // both before it has ever run and when its last attempt was
8038            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8039            // permission to merge on nothing but the review record. Only a
8040            // gate that actually ran may proceed; an intentional local
8041            // review can also finish with verification explicitly not configured.
8042            // `RunState::gate_status` computes that distinction.
8043            || !self.state.gate_allows_progress()
8044        {
8045            return Ok(());
8046        }
8047        // This node's own record, not `status`: `status == Ready` is not
8048        // unique to the harmless `MergeMode::None` path this line was
8049        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8050        // run's PR was closed without merging — and on that run `mode` is
8051        // still `Pr`, so a reentry that fell through here would push and
8052        // open a second pull request. `self.state.merge` is set exactly once
8053        // this node (or `land`) has already produced a verdict, under every
8054        // mode, which is what "already done" actually means here.
8055        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8056            return Ok(());
8057        }
8058        let Some(winner) = self.state.winner().cloned() else {
8059            return Ok(());
8060        };
8061        let repo = self.state.repo.clone();
8062        let base = self.state.base_branch.clone();
8063        let mode = self.state.config.merge.mode;
8064        let style = self.state.config.merge.style;
8065        let facts = if is_review_run(&self.state) {
8066            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8067            let start = review_base(
8068                &repo,
8069                &self.state.config.merge.remote,
8070                &base,
8071                &self.state.base_commit,
8072                &winner.branch,
8073            )
8074            .await;
8075            branch_facts(&repo, &start, &winner.branch).await
8076        } else {
8077            None
8078        };
8079        // `None` when the base could not be freshly read: then an adopted
8080        // pull request's title is left alone.
8081        let leaked = if is_review_run(&self.state) {
8082            leaked_subjects(&self.state, &winner.branch).await
8083        } else {
8084            Some(Vec::new())
8085        };
8086        // `None`: parked on the owner's word about a withheld title or body.
8087        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8088        // the resume arrives here again.
8089        let Some(pr) = self
8090            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8091            .await?
8092        else {
8093            return Ok(());
8094        };
8095        let message = pr.commit_message();
8096
8097        let outcome = match mode {
8098            MergeMode::None => MergeOutcome {
8099                mode,
8100                ok: true,
8101                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8102                empty: false,
8103            },
8104            MergeMode::Pr | MergeMode::Local
8105                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8106            {
8107                MergeOutcome {
8108                    mode,
8109                    ok: false,
8110                    detail: empty_candidate_detail(&self.state, &base),
8111                    empty: true,
8112                }
8113            }
8114            MergeMode::Local => {
8115                let on = git::current_branch(&repo).await?;
8116                if on.as_deref() != Some(base.as_str()) {
8117                    MergeOutcome {
8118                        mode,
8119                        ok: false,
8120                        detail: format!(
8121                            "{} has {} checked out, not the base branch {base}",
8122                            repo.display(),
8123                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8124                        ),
8125                        empty: false,
8126                    }
8127                } else if !git::is_clean(&repo).await? {
8128                    MergeOutcome {
8129                        mode,
8130                        ok: false,
8131                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8132                        empty: false,
8133                    }
8134                } else {
8135                    let out = match style {
8136                        MergeStyle::Merge => {
8137                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8138                        }
8139                        MergeStyle::Squash => {
8140                            git::merge_squash(&repo, &winner.branch, &message).await?
8141                        }
8142                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8143                    };
8144                    MergeOutcome {
8145                        mode,
8146                        ok: out.ok(),
8147                        detail: if out.ok() { out.stdout } else { out.stderr },
8148                        empty: false,
8149                    }
8150                }
8151            }
8152            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8153                self.state.pr_publication_pending = true;
8154                self.state.save()?;
8155                let out =
8156                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8157                self.state.pr_publication_pending = out.is_err();
8158                MergeOutcome {
8159                    mode,
8160                    ok: out.is_ok(),
8161                    detail: match out {
8162                        Ok(url) => url,
8163                        Err(e) => format!("{e:#}"),
8164                    },
8165                    empty: false,
8166                }
8167            }
8168            MergeMode::Pr => {
8169                let remote = self.state.config.merge.remote.clone();
8170                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8171                if !pushed.ok() {
8172                    MergeOutcome {
8173                        mode,
8174                        ok: false,
8175                        detail: pushed.stderr,
8176                        empty: false,
8177                    }
8178                } else {
8179                    // A retry or resume of a run whose branch already has an
8180                    // open pull request adopts it rather than failing on a
8181                    // duplicate. Only this winner branch into this base:
8182                    // `branch_for` derives the name from the run id, so a
8183                    // different run's pull request never matches.
8184                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8185                    let out = match pr_merge_plan(found) {
8186                        PrPlan::Create => {
8187                            gh_pr_create(
8188                                &winner.worktree,
8189                                &base,
8190                                &winner.branch,
8191                                &pr.title,
8192                                &pr.body,
8193                            )
8194                            .await
8195                        }
8196                        PrPlan::Adopt { url, title } => {
8197                            self.state
8198                                .event("merge", format!("Pr: adopted open pull request {url}"));
8199                            if self.state.github_text.as_ref().is_some_and(|g| {
8200                                g.resolved
8201                                    && g.choice == Some(GithubTextChoice::Rewrite)
8202                                    && g.rewrite.is_some()
8203                            }) {
8204                                // The owner chose the entire pair, even if the
8205                                // existing title already matches or was edited.
8206                                land::set_pr_text_verbatim(
8207                                    &winner.worktree,
8208                                    &url,
8209                                    &pr.title,
8210                                    &pr.body,
8211                                )
8212                                .await
8213                                .map(|()| url)
8214                            } else {
8215                                if title != pr.title
8216                                    && (!is_review_run(&self.state)
8217                                        || leaked
8218                                            .as_deref()
8219                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8220                                    && let Err(e) =
8221                                        if self.state.github_text.as_ref().is_some_and(|g| {
8222                                            g.resolved
8223                                                && g.title
8224                                                && g.choice == Some(GithubTextChoice::Original)
8225                                        }) {
8226                                            // The owner chose to post this title unmodified.
8227                                            land::set_pr_title_verbatim(
8228                                                &winner.worktree,
8229                                                &url,
8230                                                &pr.title,
8231                                            )
8232                                            .await
8233                                        } else {
8234                                            land::set_pr_title(
8235                                                &mut self.state,
8236                                                &winner.worktree,
8237                                                &url,
8238                                                &pr.title,
8239                                            )
8240                                            .await
8241                                        }
8242                                {
8243                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8244                                    self.state
8245                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8246                                }
8247                                Ok(url)
8248                            }
8249                        }
8250                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8251                    };
8252                    match out {
8253                        Ok(url) => MergeOutcome {
8254                            mode,
8255                            ok: true,
8256                            detail: url,
8257                            empty: false,
8258                        },
8259                        Err(e) => MergeOutcome {
8260                            mode,
8261                            ok: false,
8262                            detail: e.to_string(),
8263                            empty: false,
8264                        },
8265                    }
8266                }
8267            }
8268        };
8269
8270        self.state.status = match (mode, outcome.ok) {
8271            (MergeMode::None, _) => RunStatus::Ready,
8272            (_, true) if outcome.empty => RunStatus::Ready,
8273            (MergeMode::Pr, true) => RunStatus::Ready,
8274            (_, true) => RunStatus::Merged,
8275            (_, false) => RunStatus::Blocked,
8276        };
8277        self.state.event(
8278            "merge",
8279            format!(
8280                "{:?}: {}",
8281                mode,
8282                outcome.detail.lines().next().unwrap_or("")
8283            ),
8284        );
8285        self.state.merge = Some(outcome);
8286        self.state.save()?;
8287
8288        // The PR is open and the run would historically stop here, leaving the
8289        // operator to watch checks, feed review comments back to a fixer, and
8290        // merge. That was done by hand six times in one session before this
8291        // existed. Opt-in, because merging is the one irreversible thing magi
8292        // can do to a repository.
8293        if self.state.config.graph.land
8294            && mode == MergeMode::Pr
8295            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8296        {
8297            self.run_land().await?;
8298        }
8299        // `run_land` may have left `status` at `Landing` - still waiting on
8300        // CI or the owner's approval, not actually settled - so this has to
8301        // read whatever `status` ended up as here, not the `Merged` this
8302        // function set a few lines up.
8303        self.settle_questions();
8304        Ok(())
8305    }
8306
8307    /// Enter `land`.
8308    ///
8309    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8310    /// a resumed run's re-entry. `land::land` itself is what serialises the
8311    /// two git-mutating moments inside the loop — the rebase push and
8312    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8313    /// to hold a lock across the whole call, and doing so would serialise
8314    /// this run's CI wait against a *different* run's land-approval resume
8315    /// in the same repository, which is exactly the "must not wait on
8316    /// another task" property the daemon's slot-freeing exists to give.
8317    async fn run_land(&mut self) -> Result<()> {
8318        crate::pr_head::ensure_owner(&self.state)?;
8319        let url = self
8320            .state
8321            .pr
8322            .as_ref()
8323            .map(|pr| pr.url.clone())
8324            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8325            .unwrap_or_default();
8326        let url = url.lines().next().unwrap_or("").trim().to_owned();
8327        if !url.starts_with("http") {
8328            return Ok(());
8329        }
8330        // A land failure is not a lost run: the work is on a branch and the
8331        // pull request is open, which is exactly where a human takes over.
8332        loop {
8333            match land::land(&mut self.state, &url).await {
8334                Ok(_)
8335                    if self
8336                        .state
8337                        .merge_repairs
8338                        .last()
8339                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8340                {
8341                    if self.repair_merge_review().await? {
8342                        continue;
8343                    }
8344                }
8345                Ok(pr) if self.state.parked => {
8346                    // `land` already saved the parked marker; nothing here
8347                    // overrides `status` back to a terminal value while an
8348                    // approval is still outstanding.
8349                    let _ = pr;
8350                }
8351                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8352                    self.state.status = RunStatus::Superseded;
8353                    self.state.save()?;
8354                }
8355                Ok(pr) => {
8356                    self.state.status = match pr.state {
8357                        land::PrLifecycle::Merged => RunStatus::Merged,
8358                        _ => RunStatus::Blocked,
8359                    };
8360                    // Downstream of a confirmed merge only - see
8361                    // `bump::should_release_bump`'s own doc for why this one
8362                    // check covers all three of `land`'s success paths.
8363                    // Best-effort: the run already landed, so a failure here
8364                    // (the decision call, `gh`, `cargo`) is recorded and never
8365                    // turns a landed run into a failed one.
8366                    if bump::should_release_bump(self.state.status)
8367                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8368                    {
8369                        // The event is the run's own record. Not-eligible cases
8370                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8371                        // `Err` is a bump that was tried and failed:
8372                        // `after_merge` itself raises the operator notice for
8373                        // that, whether or not a release PR exists yet.
8374                        self.state
8375                            .event("bump", format!("release bump skipped: {e:#}"));
8376                    }
8377                    // Independent of the bump, and best-effort in the same way:
8378                    // findings the merge left open become follow-up tasks.
8379                    if self.state.status == RunStatus::Merged {
8380                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8381                    }
8382                    self.state.save()?;
8383                }
8384                Err(e) => {
8385                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8386                        return Err(e);
8387                    }
8388                    self.state.status = RunStatus::Blocked;
8389                    self.state.event("land", format!("gave up: {e}"));
8390                    self.state.save()?;
8391                }
8392            }
8393            break;
8394        }
8395        Ok(())
8396    }
8397
8398    /// One repair cycle stays under the existing driver/task claim. An
8399    /// interrupted fixer is rolled back and held rather than run twice.
8400    async fn repair_merge_review(&mut self) -> Result<bool> {
8401        use crate::run::MergeRepairPhase as Phase;
8402        crate::pr_head::ensure_owner(&self.state)?;
8403        let i = self.state.merge_repairs.len() - 1;
8404        let request = self.state.merge_repairs[i].clone();
8405        if request.phase == Phase::Fixing {
8406            let restored = match self.state.winner().cloned() {
8407                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8408                None => Err(anyhow::anyhow!("repair has no winning branch")),
8409            };
8410            let why = match restored {
8411                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8412                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8413                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8414            };
8415            self.hold_merge_repair(i, why)?;
8416            return Ok(false);
8417        }
8418        let failure = if i >= self.state.config.graph.land_rounds
8419            || self.state.config.graph.review_rounds == 0
8420        {
8421            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8422        } else {
8423            None
8424        };
8425        if let Some(why) = failure {
8426            self.hold_merge_repair(i, why)?;
8427            return Ok(false);
8428        }
8429        if request.phase == Phase::Requested {
8430            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8431                Ok(pr) => pr,
8432                Err(e) => {
8433                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8434                    return Ok(false);
8435                }
8436            };
8437            self.state.merge_repairs[i].phase = Phase::Fixing;
8438            self.state.land_approval = None;
8439            self.state.land_armed_head = None;
8440            self.state.save()?;
8441            // Serialize the exact records, including finding ids/details,
8442            // reconsideration reasons, final votes and verification output.
8443            let evidence = serde_json::to_string_pretty(&request)?;
8444            let reason = format!(
8445                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8446            );
8447            let budget = self.state.config.graph.land_rounds;
8448            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8449            match result {
8450                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8451                Ok(land::Fixed::Failed(why)) => {
8452                    self.hold_merge_repair(
8453                        i,
8454                        format!(
8455                            "repair failed: {why}; inspect the branch and run an explicit review"
8456                        ),
8457                    )?;
8458                    return Ok(false);
8459                }
8460                Err(e) => {
8461                    self.hold_merge_repair(
8462                        i,
8463                        format!(
8464                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8465                        ),
8466                    )?;
8467                    return Ok(false);
8468                }
8469            }
8470            // Even no change gets a real new reviewer wave. The previous
8471            // evidence remains in the request; a former clean/concluded round
8472            // cannot short-circuit the new review or gate.
8473            self.begin_merge_repair_review(i)?;
8474        }
8475        self.review_loop().await?;
8476        self.gate().await?;
8477        if self.resource_halt()? || self.state.parked {
8478            return Ok(false);
8479        }
8480        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8481            != Some(RunStatus::Gating)
8482            || !self.state.gate_allows_progress()
8483        {
8484            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8485            return Ok(false);
8486        }
8487        // Review and gate may have added commits after the initial repair.
8488        // Publish exactly their final head to the existing PR, with a lease.
8489        let winner = self
8490            .state
8491            .winner()
8492            .cloned()
8493            .context("repair has no winning branch")?;
8494        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8495        if self.state.gate_head.as_deref() != Some(after.as_str())
8496            || !git::is_clean(&winner.worktree).await?
8497        {
8498            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8499            return Ok(false);
8500        }
8501        let expected = self
8502            .state
8503            .pr_pushed_head
8504            .as_deref()
8505            .unwrap_or(&request.head)
8506            .to_owned();
8507        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8508        // Push the checked commit itself: a local ref moving after the check
8509        // must not change which commit the forge receives.
8510        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8511        let refspec = format!("{after}:refs/heads/{public}");
8512        let push = git::git_raw(
8513            &winner.worktree,
8514            &["push", &lease, &self.state.config.merge.remote, &refspec],
8515        )
8516        .await;
8517        match push {
8518            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8519            Ok(out) => {
8520                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8521                return Ok(false);
8522            }
8523            Err(e) => {
8524                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8525                return Ok(false);
8526            }
8527        }
8528        self.state.merge_repairs[i].phase = Phase::Complete;
8529        self.state.status = RunStatus::Landing;
8530        self.state.event(
8531            "repair",
8532            "new review and gate completed; fresh owner merge decision required",
8533        );
8534        self.state.save()?;
8535        Ok(true)
8536    }
8537
8538    fn merge_repair_artifacts(&self) -> PathBuf {
8539        let artifacts = agent::artifacts_dir(&self.state.dir());
8540        match self
8541            .state
8542            .merge_repairs
8543            .last()
8544            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8545        {
8546            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8547            None => artifacts,
8548        }
8549    }
8550
8551    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8552        self.state.reviews.clear();
8553        self.state.gate.clear();
8554        self.state.gate_ran = false;
8555        self.state.gate_head = None;
8556        self.state.gate_fixes.clear();
8557        self.state.gate_repair_e2e = None;
8558        self.state.pre_gate.clear();
8559        self.state.pre_gate_commit = None;
8560        self.state.contested_handoff = None;
8561        self.state.land_reruns = None;
8562        self.state.land_approval = None;
8563        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8564        self.state.status = RunStatus::Reviewing;
8565        self.state.save()
8566    }
8567
8568    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8569        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8570        self.state.merge_repairs[index].failure = Some(why.clone());
8571        self.state.status = RunStatus::Blocked;
8572        self.state.event("repair", why);
8573        self.state.save()
8574    }
8575
8576    // -------------------------------------------------------------- helpers
8577
8578    /// Fetch or create a seat, keeping its conversation across nodes.
8579    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8580        if let Some(existing) = self.state.seats.get(key)
8581            && existing.agent == agent
8582        {
8583            return existing.clone();
8584        }
8585        // A seat that changes agent mints its session id from the agent too,
8586        // like a handover: the old agent's uuid is already taken by the CLI.
8587        let fresh = if self.state.seats.contains_key(key) {
8588            handover_seat(key, agent, self.state.next_seat_seed())
8589        } else {
8590            SeatState::new(key, agent, self.state.seed)
8591        };
8592        self.state.seats.insert(key.to_owned(), fresh.clone());
8593        fresh
8594    }
8595
8596    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8597    /// seat to another roster agent, in which case that agent. Nodes that
8598    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8599    /// reconsideration) must keep talking to whoever answered it, not slip
8600    /// back to the agent that failed it.
8601    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8602        match self.state.seats.get(key) {
8603            Some(s) if s.agent != spec.id => {
8604                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8605            }
8606            _ => spec,
8607        }
8608    }
8609
8610    /// A candidate rendered for judging, with the leak policy applied.
8611    fn view(&self, c: &Candidate) -> CandidateView {
8612        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8613            .unwrap_or_default();
8614        let (patch, _) = blind::sanitize_patch(
8615            &format!("candidate {} patch", c.label),
8616            &raw,
8617            &self.state.config.blind,
8618        );
8619        CandidateView {
8620            label: c.label,
8621            branch: c.branch.clone(),
8622            summary: c.summary.clone(),
8623            stat: c.stat.clone(),
8624            patch,
8625        }
8626    }
8627
8628    /// The full candidate set as prompt text, for seats with no live session.
8629    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8630        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8631        prompt::judge(
8632            "(see above)",
8633            &views,
8634            self.roles.judges.len(),
8635            base_short,
8636            "en",
8637        )
8638    }
8639
8640    /// The final-vote prompt with everything a seat that has no session of its
8641    /// own needs: the candidates, the seat's own ranking and reasons, and the
8642    /// anonymised deliberation it took part in (only when there was one, so a
8643    /// handed-over seat never sees more than the seat it replaces did). The
8644    /// `Final vote` heading stays first.
8645    fn vote_prompt_full(
8646        &self,
8647        j: usize,
8648        labels: &[char],
8649        language: &str,
8650        candidates: &[Candidate],
8651        base_short: &str,
8652    ) -> String {
8653        let mut text = format!(
8654            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8655            prompt::final_vote(labels, language),
8656            self.state.instruction,
8657            self.candidate_block(candidates, base_short)
8658        );
8659        if let Some(own) = self
8660            .state
8661            .judgements
8662            .get(j)
8663            .filter(|r| !r.ranking.is_empty())
8664        {
8665            let reasons = own
8666                .reasons
8667                .iter()
8668                .map(|(k, v)| format!("- {k}: {v}"))
8669                .collect::<Vec<_>>()
8670                .join("\n");
8671            text.push_str(&format!(
8672                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8673                own.ranking.iter().collect::<String>(),
8674                if reasons.is_empty() {
8675                    ""
8676                } else {
8677                    ", because:\n"
8678                }
8679            ));
8680        }
8681        if !self.state.deliberation.is_empty() {
8682            text.push_str("\n# What was argued before this vote\n");
8683            for t in self.transcript(&[], j) {
8684                text.push_str(&format!(
8685                    "\n## {}{}\n\n{}\n",
8686                    t.who,
8687                    if t.is_self { " (you)" } else { "" },
8688                    t.body.trim()
8689                ));
8690            }
8691        }
8692        text
8693    }
8694
8695    /// Anonymised transcript for judge `self_idx`.
8696    ///
8697    /// The initial rankings are always the opening statements. Seeding them
8698    /// only when no turn had been taken yet meant every judge after the first
8699    /// argued against a single voice instead of against the actual split — the
8700    /// disagreement is the information, so it is always on the table.
8701    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8702        let mut turns = Vec::new();
8703        for j in &self.state.judgements {
8704            if j.ranking.is_empty() {
8705                continue;
8706            }
8707            let reasons = j
8708                .reasons
8709                .iter()
8710                .map(|(k, v)| format!("- {k}: {v}"))
8711                .collect::<Vec<_>>()
8712                .join("\n");
8713            turns.push(Turn {
8714                who: format!("Judge {} (opening ranking)", j.judge),
8715                is_self: j.judge == self_idx + 1,
8716                body: format!(
8717                    "Ranked {}{}{reasons}",
8718                    j.ranking.iter().collect::<String>(),
8719                    if reasons.is_empty() {
8720                        ""
8721                    } else {
8722                        ", because:\n"
8723                    }
8724                ),
8725            });
8726        }
8727        for t in self
8728            .state
8729            .deliberation
8730            .iter()
8731            .flat_map(|r| r.turns.iter())
8732            .chain(current)
8733        {
8734            turns.push(Turn {
8735                who: format!("Judge {}", t.judge),
8736                is_self: t.judge == self_idx + 1,
8737                body: t.body.clone(),
8738            });
8739        }
8740        turns
8741    }
8742}
8743
8744/// Does this seat still hold the context a follow-up prompt would rely on?
8745fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8746    agent::has_session(spec.kind, seat, sessions)
8747}
8748
8749/// The next entry in `roster` after `start`, never wrapping back to the
8750/// front, whose id is not in `tried` yet.
8751///
8752/// Starts one past `start` rather than at the front of `roster`: `start` is
8753/// the seat's own original position, and a seat whose candidate slot already
8754/// sits on the roster's second entry must fall through to the third next, not
8755/// restart at the first — which is very likely a different candidate's own
8756/// agent already. Never wraps back past `start`, for the same reason: an
8757/// entry earlier in the roster than the seat's own position is almost
8758/// certainly some *other* candidate slot's own agent, and once the tail of
8759/// the roster is exhausted there are no more untried agents for *this* seat
8760/// to fall through to — the caller's fallback chain ends there, exactly as
8761/// "no further untried agents remain in the list for that seat" asks for.
8762///
8763/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8764/// the same id twice (an operator's `roles.implementers` typo, or a
8765/// `[[agents]]` list reused across roles) must not let
8766/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8767/// over `roster` either finds an untried id or runs out, so this always
8768/// terminates regardless of duplicates.
8769fn next_untried_in_roster<'a>(
8770    roster: &'a [AgentSpec],
8771    start: usize,
8772    tried: &BTreeSet<String>,
8773) -> Option<&'a AgentSpec> {
8774    roster
8775        .get(start + 1..)?
8776        .iter()
8777        .find(|s| !tried.contains(&s.id))
8778}
8779
8780/// The successor for a seat, shared by all four seat kinds (implement, judge,
8781/// review, advise). `others` holds the ids that *currently* occupy the other
8782/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8783/// sees them), so roster entries beyond the seat count act as spares: a failed
8784/// seat goes to an agent no other seat holds whenever the roster permits.
8785///
8786/// `carried` is `Some` only for the review loop's carried failure history.
8787/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8788/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8789/// whole roster: untried and not another seat's occupant; (3) the plain walk
8790/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8791/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8792/// a better outcome than an empty seat, and it keeps the answer to "is there
8793/// a successor at all" exactly what it was before occupants were considered,
8794/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8795/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8796/// carried failure is a cheaper bet than doubling an agent on the panel, and
8797/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8798/// the same round are handled one at a time, so a seat later in the batch
8799/// sees an earlier one's new occupant but not yet its own freed agent.
8800fn pick_successor<'a>(
8801    roster: &'a [AgentSpec],
8802    start: usize,
8803    tried: &BTreeSet<String>,
8804    carried: Option<&BTreeSet<String>>,
8805    others: &BTreeSet<String>,
8806) -> Option<&'a AgentSpec> {
8807    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8808    roster
8809        .get(start + 1..)
8810        .and_then(|tail| {
8811            tail.iter()
8812                .filter(free)
8813                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8814        })
8815        .or_else(|| {
8816            carried
8817                .is_some()
8818                .then(|| roster.iter().find(free))
8819                .flatten()
8820        })
8821        .or_else(|| match carried {
8822            Some(c) => next_for_seat(roster, start, tried, c),
8823            None => next_untried_in_roster(roster, start, tried),
8824        })
8825}
8826
8827/// The next agent for a seat that carries its failure history across rounds
8828/// (the review loop). `round_tried` is this round's own bound and starts
8829/// empty every round; `carried_failed` only decides priority.
8830///
8831/// First: an id walking forward from `start`, never wrapping, that is neither
8832/// tried this round nor failed in an earlier one. Only when that is exhausted
8833/// does it rescue: the first roster id (in roster order, so this one *does*
8834/// look before `start`) not yet tried this round, which is by then a carried
8835/// failure. Each id is rescued at most once per round, so it cannot loop.
8836fn next_for_seat<'a>(
8837    roster: &'a [AgentSpec],
8838    start: usize,
8839    round_tried: &BTreeSet<String>,
8840    carried_failed: &BTreeSet<String>,
8841) -> Option<&'a AgentSpec> {
8842    roster
8843        .get(start + 1..)?
8844        .iter()
8845        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8846        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8847}
8848
8849/// Where a reviewer seat starts a round: the agent that last answered it when
8850/// it is still on the roster and not marked failed, else the spec's own agent
8851/// unless it failed, else the next roster agent that has not failed, else the
8852/// spec's own agent again (the whole roster failed). Ids no longer on the
8853/// roster are ignored. An empty roster has no handover, so the spec stands.
8854fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8855    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8856        return spec;
8857    };
8858    let ok = |id: &str| !h.failed.contains(id);
8859    if let Some(last) = h.last_ok.as_deref()
8860        && ok(last)
8861        && let Some(s) = roster.iter().find(|s| s.id == last)
8862    {
8863        return s.clone();
8864    }
8865    if ok(&spec.id) {
8866        return spec;
8867    }
8868    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8869    roster
8870        .get(from + 1..)
8871        .into_iter()
8872        .flatten()
8873        .chain(roster.iter())
8874        .find(|s| ok(&s.id))
8875        .cloned()
8876        .unwrap_or(spec)
8877}
8878
8879/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
8880/// seed so a CLI that mints its session id up front (`--session-id`) never
8881/// reuses the uuid the previous agent already opened under the same seat key.
8882pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8883    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8884}
8885
8886/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
8887/// so the classifier below and the code that builds the message cannot drift.
8888const TIMED_OUT: &str = "timed out";
8889
8890impl FailClass {
8891    /// `None` for an answer; otherwise how the turn failed.
8892    fn of(out: &AgentOutcome) -> Option<Self> {
8893        match out {
8894            AgentOutcome::Ok(_) => None,
8895            AgentOutcome::Quota(_) => Some(Self::Quota),
8896            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8897            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8898            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8899            // Not a failure of the agent. Callers check for it first; `None`
8900            // here means "no class to hand over on".
8901            AgentOutcome::ResourceBlocked(_) => None,
8902        }
8903    }
8904
8905    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
8906    fn stem_word(&self) -> &'static str {
8907        match self {
8908            Self::Quota => "quota",
8909            _ => "handover",
8910        }
8911    }
8912}
8913
8914/// The message's first line with its variable parts removed — digit runs and
8915/// path-like tokens — so "exit code 2" and "exit code 7" read
8916/// as one kind of failure.
8917fn failure_signature(msg: &str) -> String {
8918    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8919    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
8920    // the historical machine key while the process diagnosis uses readable prose.
8921    if let Some(code) = line
8922        .strip_prefix("process exited with no usable output (exit code ")
8923        .and_then(|s| s.strip_suffix(')'))
8924        .and_then(|s| s.parse::<i32>().ok())
8925    {
8926        line = format!("exited with some({code}) and no usable output");
8927    } else if matches!(
8928        line.as_str(),
8929        "process exited with no usable output (no exit code reported)"
8930            | "process exited with no usable output (exit code unavailable)"
8931    ) {
8932        line = "exited with none and no usable output".to_owned();
8933    }
8934    let mut out = Vec::new();
8935    for word in line.split_whitespace() {
8936        if word.contains('/') || word.contains('\\') {
8937            out.push("<path>".to_owned());
8938            continue;
8939        }
8940        let mut w = String::new();
8941        let mut in_digits = false;
8942        for c in word.chars() {
8943            if c.is_ascii_digit() {
8944                if !in_digits {
8945                    w.push('#');
8946                }
8947                in_digits = true;
8948            } else {
8949                in_digits = false;
8950                w.push(c);
8951            }
8952        }
8953        out.push(w);
8954    }
8955    out.join(" ").chars().take(120).collect()
8956}
8957
8958/// Whether a seat that just failed with `cur` may go to the next roster agent.
8959/// A quota or a timeout always may. Any other failure may not when the agent
8960/// before it failed the same way: an error the prompt causes would otherwise
8961/// walk the whole roster. `prev` is the class of the immediately preceding
8962/// agent's failure, so a quota or timeout in between breaks the run of
8963/// identical failures by itself.
8964fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8965    match cur {
8966        FailClass::Quota | FailClass::Timeout => true,
8967        FailClass::Other(_) => prev != Some(cur),
8968    }
8969}
8970
8971/// A short human reason for a failed outcome, for the handover record.
8972fn fail_reason(out: &AgentOutcome) -> String {
8973    match out {
8974        AgentOutcome::Ok(_) => String::new(),
8975        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8976        AgentOutcome::Dropped(o) => format!(
8977            "the CLI dropped the stream ({})",
8978            o.dropped
8979                .as_ref()
8980                .map(|d| d.why.as_str())
8981                .unwrap_or("it ended without delivering its answer")
8982        ),
8983        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8984            e.lines().next().unwrap_or("").chars().take(160).collect()
8985        }
8986    }
8987}
8988
8989/// Note one handover in the run: the structured record and, in the timeline,
8990/// the sentence a person reads. A quota keeps the wording it always had.
8991pub(crate) fn record_handover(
8992    state: &mut RunState,
8993    node: &str,
8994    seat: &str,
8995    from: &str,
8996    to: &str,
8997    class: &FailClass,
8998    reason: &str,
8999) {
9000    let message = if *class == FailClass::Quota {
9001        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9002    } else {
9003        format!("{seat}: handed over {from} -> {to} ({reason})")
9004    };
9005    state.event(node, message);
9006    state.handovers.push(Handover {
9007        at: Timestamp::now(),
9008        node: node.to_owned(),
9009        seat: seat.to_owned(),
9010        from: from.to_owned(),
9011        to: to.to_owned(),
9012        reason: reason.to_owned(),
9013    });
9014}
9015
9016/// Did this reply report running a command whose own CLI never confirmed an
9017/// exit status?
9018///
9019/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9020/// command *finished* (see that type's own doc), so this can only be `true`
9021/// for a command whose completion event carried no readable exit code — not
9022/// for one that simply is not mentioned at all. That is the one signal this
9023/// crate can read, from the same record `state.jobs` renders, about a reply
9024/// standing next to work its own CLI cannot vouch for finishing; it is
9025/// deliberately not a check on the exit code's *value* (a fixer legitimately
9026/// runs a command that fails mid-iteration before it succeeds) and not a
9027/// guess at a command still running in the background (which emits no event
9028/// at all, and so leaves no evidence here to find).
9029fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9030    commands.iter().any(|c| c.exit_code.is_none())
9031}
9032
9033/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9034/// trusted as a verified no-op — the adoption guard's own text-level half.
9035///
9036/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9037/// timed out): a marker only earns the benefit of the doubt from a turn the
9038/// CLI itself vouches for finishing properly, the same house style
9039/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9040/// *fix* report to for `commands`. A candidate that timed out, exited
9041/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9042/// is, whatever prose it wrote — this returns `None` before it ever looks at
9043/// `text`. The remaining guards (the tree really is empty, the evidence is
9044/// non-empty) are the caller's: this only reads what the reply *claimed*.
9045fn verified_noop_claim(
9046    usable: bool,
9047    commands: &[agent::CommandEvidence],
9048    text: &str,
9049) -> Option<String> {
9050    (usable && !has_unconfirmed_command(commands))
9051        .then(|| verdict::verified_noop(text))
9052        .flatten()
9053}
9054
9055fn short(commit: &str) -> String {
9056    commit.chars().take(7).collect()
9057}
9058
9059fn make_executable(path: &Path) -> Result<()> {
9060    #[cfg(unix)]
9061    {
9062        use std::os::unix::fs::PermissionsExt as _;
9063        let mut perms = std::fs::metadata(path)?.permissions();
9064        perms.set_mode(0o755);
9065        std::fs::set_permissions(path, perms)?;
9066    }
9067    #[cfg(not(unix))]
9068    {
9069        let _ = path;
9070    }
9071    Ok(())
9072}
9073
9074/// What every seat in one batch shares: where the answers are attributed, the
9075/// prompt overlay they inherit, and the build cache they are told to use.
9076///
9077/// A struct rather than four more parameters: `wave` also needs the run's
9078/// state (to record who is answering right now) and the attempt number, and
9079/// eight positional arguments is both unreadable and a clippy error.
9080struct WaveCtx<'a> {
9081    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9082    /// paid for it.
9083    run: &'a str,
9084    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9085    node: &'a str,
9086    prompts: &'a Prompts,
9087    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9088    cache: Option<&'a Path>,
9089    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9090    /// `JobRecord::round`. `None` for every other node.
9091    round: Option<usize>,
9092    /// Carry each seat's failed-agent history across waves (the review loop
9093    /// only): start-of-round priority and handover choice read
9094    /// [`RunState::seat_history`], and every answer or failure writes it.
9095    carry_seats: bool,
9096    /// The run's park / interrupt requests, so a wait for a shared resource
9097    /// can be abandoned the moment the operator asks to park.
9098    pauses: Option<(&'a Pause, &'a Pause)>,
9099}
9100
9101/// Shared scheduling policy for operations and spawned seat invocations.
9102fn repository_wait(
9103    state: &RunState,
9104    pauses: Option<(&Pause, &Pause)>,
9105) -> crate::repo_lock::WaitPolicy {
9106    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9107    let inherited = crate::repo_lock::WaitPolicy::current();
9108    crate::repo_lock::WaitPolicy::new(
9109        Duration::from_secs(state.config.graph.timeout_resource_wait),
9110        move || {
9111            inherited.cancelled()
9112                || pauses
9113                    .as_ref()
9114                    .is_some_and(|(a, b)| a.parked() || b.parked())
9115        },
9116    )
9117}
9118
9119fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9120    if error.is::<crate::repo_lock::ResourceBlocked>() {
9121        state.resource_block = Some(crate::run::ResourceBlock {
9122            node: "repository".into(),
9123            kind: if error.to_string().contains("disk pressure") {
9124                "disk pressure"
9125            } else {
9126                "repository coordination"
9127            }
9128            .into(),
9129            detail: error.to_string(),
9130            at: Timestamp::now(),
9131        });
9132        state.event("resource", error.to_string());
9133    }
9134}
9135
9136async fn coordinate_operation(
9137    state: &mut RunState,
9138    cwd: &Path,
9139    wait: &crate::repo_lock::WaitPolicy,
9140) -> Result<Option<crate::repo_lock::SeatLease>> {
9141    let result =
9142        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9143            .await;
9144    if let Err(e) = &result {
9145        record_repository_block(state, e);
9146    }
9147    result
9148}
9149
9150/// Run one job, honouring the parallelism budget.
9151async fn run_one(
9152    job: SeatJob,
9153    sem: Arc<Semaphore>,
9154    ctx: &WaveCtx<'_>,
9155    state: &mut RunState,
9156    attempt: usize,
9157) -> (SeatState, AgentOutcome) {
9158    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9159        .await
9160        .pop()
9161        .expect("one job in, one result out");
9162    (seat, out)
9163}
9164
9165/// Run every job concurrently, capped by the semaphore, preserving order.
9166///
9167/// Every seat in the batch is recorded into [`RunState::active`] before the
9168/// wave starts and cleared as each answer lands, so the run's own record says
9169/// who is still being waited on rather than only who finished.
9170async fn wave(
9171    jobs: Vec<SeatJob>,
9172    sem: Arc<Semaphore>,
9173    ctx: &WaveCtx<'_>,
9174    state: &mut RunState,
9175    attempt: usize,
9176) -> Vec<(usize, SeatState, AgentOutcome)> {
9177    let WaveCtx {
9178        run,
9179        node,
9180        prompts,
9181        cache,
9182        round,
9183        carry_seats: _,
9184        pauses,
9185    } = *ctx;
9186    for job in &jobs {
9187        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9188    }
9189    if let Err(e) = state.save() {
9190        // A failed persist of "who is answering right now" must not abort the
9191        // wave: the seats are already being asked, and the alternative is
9192        // losing the answers to save a status line nobody may even be
9193        // watching.
9194        tracing::warn!("could not persist in-progress seats: {e:#}");
9195    }
9196    // Hold the shared build cache's lease for the whole batch, not per job:
9197    // several candidates (an implement wave) or a fixer legitimately share
9198    // one cache concurrently within this run, and that stays untouched — a
9199    // single lease taken once for the whole wave and released once it is
9200    // done is what stops a *different* borrower (another run's own wave, its
9201    // e2e/gate, a human's `magi review`) from interleaving a build into the
9202    // same directory while this one is in flight. Best-effort, not
9203    // all-or-nothing: a wave that cannot get the lease within its own
9204    // longest job's budget still runs — an hour of paid implementer calls is
9205    // not thrown away over cache contention — but every write-allowed seat
9206    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9207    // below), the same fallback a read-only seat always gets, rather than
9208    // building into a directory this run was never granted. The identity
9209    // record is still invalidated below either way, so the next tracked
9210    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9211    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9212    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9213    // never borrowed from a job's agent budget (and never charged to it either,
9214    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9215    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9216    let cache_guard = if let Some(cache_dir) = cache {
9217        if jobs_had_a_writer {
9218            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9219            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9220            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9221            {
9222                Ok(Some(g)) => Some(g),
9223                // Parking: leave without spawning anything. The run parks at
9224                // the next boundary and the seats are asked afresh on resume.
9225                Ok(None) => {
9226                    state.resource_block = Some(crate::run::ResourceBlock {
9227                        node: node.to_owned(),
9228                        kind: "shared build cache".to_owned(),
9229                        detail: "the run is parking".to_owned(),
9230                        at: Timestamp::now(),
9231                    });
9232                    return block_wave(
9233                        jobs,
9234                        state,
9235                        node,
9236                        attempt,
9237                        "parked while waiting for the shared build cache",
9238                    );
9239                }
9240                // An unreadable lease is not contention: keep the old
9241                // fallback (no shared cache for this wave's seats).
9242                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9243                Err(e) => {
9244                    // Writing seats may not build into a cache this run does
9245                    // not hold, and an agent that cannot be given one is not
9246                    // asked at all: the run stops resumably instead.
9247                    let why = format!("{e:#}");
9248                    state.resource_block = Some(crate::run::ResourceBlock {
9249                        node: node.to_owned(),
9250                        kind: "shared build cache".to_owned(),
9251                        detail: why.clone(),
9252                        at: Timestamp::now(),
9253                    });
9254                    state.event(
9255                        "resource",
9256                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9257                    );
9258                    return block_wave(jobs, state, node, attempt, &why);
9259                }
9260            }
9261        } else {
9262            None
9263        }
9264    } else {
9265        None
9266    };
9267    // Every job keeps its whole budget: the wait above was not the agent's.
9268    // Restart the seats' clocks so "started" means "being spawned now".
9269    state.seats_spawning(node, attempt);
9270    if let Err(e) = state.save() {
9271        tracing::warn!("could not persist the start of a wave: {e:#}");
9272    }
9273    // Queueing behind this wave's own seats for a parallelism slot is normal
9274    // (each of them is bounded by its own agent timeout); only a wait beyond
9275    // all of them plus the resource limit means the slots never came back.
9276    let slot_limit = resource_limit
9277        + jobs
9278            .iter()
9279            .map(|j| j.timeout)
9280            .max()
9281            .unwrap_or_default()
9282            .saturating_mul(jobs.len() as u32);
9283    let mut set = tokio::task::JoinSet::new();
9284    // Seats announce themselves here the moment they hold a parallelism
9285    // permit, i.e. just before their CLI is spawned.
9286    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9287    let overlay = prompts.overlay(node);
9288    let repository_wait = repository_wait(state, pauses);
9289    for (i, mut job) in jobs.into_iter().enumerate() {
9290        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9291        if cache.is_some() {
9292            job.prompt.push('\n');
9293            job.prompt
9294                .push_str(&prompt::build_cache_note(node, job.allow_write));
9295        }
9296        let sem = Arc::clone(&sem);
9297        let run = run.to_owned();
9298        let node = node.to_owned();
9299        // Only implementers were told about the task's attachments, so only
9300        // their seats get the directory widened for reading.
9301        let attachments = if node == "implement" {
9302            state.attachments.clone()
9303        } else {
9304            Vec::new()
9305        };
9306        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9307        // `prompt::build_cache_note`'s doc for why setting it anyway is
9308        // exactly how a sandboxed reviewer's write refusal got reported as a
9309        // defect in the patch, not a property of its own seat. And a
9310        // write-allowed one is handed it only when the lease above was
9311        // actually acquired: a wave that could not get it (`cache_guard` is
9312        // `None`, see its own comment) must not send seats to build into a
9313        // directory this run does not hold - that is the exact concurrent,
9314        // unmanaged-write race this module exists to prevent, not something
9315        // "proceeding anyway" is allowed to reintroduce.
9316        let cache = cache
9317            .filter(|_| job.allow_write && cache_guard.is_some())
9318            .map(Path::to_path_buf);
9319        let spawned_tx = spawned_tx.clone();
9320        let repository_wait = repository_wait.clone();
9321        let coordination = state.config.coordination.clone();
9322        let fetched_base = state.fetched_base.clone();
9323        set.spawn(async move {
9324            // A wait for a parallelism slot is scheduling, not the agent: it
9325            // is bounded by its own limit and ends as a resource block, never
9326            // as a timeout of the agent that was not started.
9327            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9328                Ok(p) => p,
9329                Err(_) => {
9330                    return (
9331                        i,
9332                        job.seat,
9333                        AgentOutcome::ResourceBlocked(format!(
9334                            "no parallel slot became free within {}s",
9335                            slot_limit.as_secs()
9336                        )),
9337                        false,
9338                    );
9339                }
9340            };
9341            spawned_tx.send(job.seat.key.clone()).ok();
9342            let mut seat = job.seat;
9343            let out = repository_wait
9344                .scope(agent::invoke(
9345                    &job.spec,
9346                    &mut seat,
9347                    &Invocation {
9348                        scope: crate::agent::InvocationScope::Repository,
9349                        coordination: coordination.as_ref(),
9350                        fetched_base: fetched_base.as_ref(),
9351                        cwd: &job.cwd,
9352                        prompt: &job.prompt,
9353                        timeout: job.timeout,
9354                        allow_write: job.allow_write,
9355                        unsandboxed: false,
9356                        sessions: job.sessions,
9357                        artifacts: &job.artifacts,
9358                        stem: &job.stem,
9359                        run: &run,
9360                        node: &node,
9361                        cache_dir: cache.as_deref(),
9362                        attachments: &attachments,
9363                        writable: &[],
9364                    },
9365                ))
9366                .await;
9367            let interrupted = out
9368                .as_ref()
9369                .err()
9370                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9371            let out = match out {
9372                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9373                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9374                // Billed work the CLI failed to hand over is not an ordinary
9375                // failure, but its text is the CLI's raw error JSON, not an
9376                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9377                // read it as one by forgetting to check. `usable()` is always
9378                // false here (dropped implies an empty response), so this has
9379                // to be checked before the catch-all `Failed` below or the
9380                // one shape this exists for is lost with the rest.
9381                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9382                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9383                Ok(o) => AgentOutcome::Failed(format!(
9384                    "process exited with no usable output ({})",
9385                    crate::notices::exit_status(o.exit_code)
9386                )),
9387                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9388                    AgentOutcome::ResourceBlocked(e.to_string())
9389                }
9390                Err(e) => AgentOutcome::Failed(e.to_string()),
9391            };
9392            (i, seat, out, interrupted)
9393        });
9394    }
9395    drop(spawned_tx);
9396    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9397    loop {
9398        let joined = tokio::select! {
9399            Some(key) = spawned_rx.recv() => {
9400                // The clock of this seat starts now, not when it queued.
9401                state.seat_spawned(&key);
9402                if let Err(e) = state.save() {
9403                    tracing::warn!("could not persist a seat's start: {e:#}");
9404                }
9405                continue;
9406            }
9407            joined = set.join_next() => joined,
9408        };
9409        let Some(joined) = joined else {
9410            break;
9411        };
9412        let (i, seat, out, interrupted) = match joined {
9413            Ok(v) => v,
9414            // No seat to clear: a panicked task never reported which one it
9415            // was. The defensive sweep below this loop is what stops that
9416            // seat's `active` entry from surviving forever.
9417            Err(e) => {
9418                tracing::error!("agent task panicked: {e}");
9419                continue;
9420            }
9421        };
9422        state.seat_finished(&seat.key);
9423        if interrupted
9424            && node == "implement"
9425            && let Some(label) = state
9426                .candidates
9427                .iter()
9428                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9429                .map(|candidate| candidate.label)
9430        {
9431            state.candidate_preservation.insert(
9432                label,
9433                "interrupted implementation awaiting persistence".into(),
9434            );
9435            state.interrupted_implementations.insert(label);
9436        }
9437        if let AgentOutcome::ResourceBlocked(why) = &out
9438            && state.resource_block.is_none()
9439        {
9440            state.resource_block = Some(crate::run::ResourceBlock {
9441                node: node.to_owned(),
9442                kind: if why.contains("disk pressure") {
9443                    "disk pressure"
9444                } else if why.contains("repository coordination") {
9445                    "repository coordination"
9446                } else {
9447                    "parallel slot"
9448                }
9449                .to_owned(),
9450                detail: why.clone(),
9451                at: Timestamp::now(),
9452            });
9453            let activity = if interrupted {
9454                "agent execution interrupted"
9455            } else {
9456                "no agent was started"
9457            };
9458            state.event("resource", format!("{node}: {activity} - {why}"));
9459        }
9460        record_jobs(state, node, round, &seat.key, &out);
9461        if let Err(e) = state.save() {
9462            tracing::warn!("could not persist a seat's completion: {e:#}");
9463        }
9464        if collected.len() <= i {
9465            collected.resize_with(i + 1, || None);
9466        }
9467        collected[i] = Some((i, seat, out));
9468    }
9469    // Belt-and-braces for the panic branch above: every seat this exact batch
9470    // started shares this `(node, attempt)` pair, and every seat that finished
9471    // normally already cleared itself, so anything left tagged with it here
9472    // can only be a panicked task's leftover. Cleared unconditionally rather
9473    // than left to read as still answering forever.
9474    if state
9475        .active
9476        .values()
9477        .any(|a| a.node == node && a.attempt == attempt)
9478    {
9479        state
9480            .active
9481            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9482        if let Err(e) = state.save() {
9483            tracing::warn!("could not persist the end of a wave: {e:#}");
9484        }
9485    }
9486    // Whether or not the lease above was actually held, several worktrees
9487    // may just have built into the cache with nothing here able to name one
9488    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9489    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9490    // clean it might not have strictly needed; trusting a stale match would
9491    // cost it a wrong answer.
9492    if let Some(cache_dir) = cache
9493        && jobs_had_a_writer
9494    {
9495        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9496    }
9497    if let Some(guard) = cache_guard {
9498        guard.release();
9499    }
9500    collected.into_iter().flatten().collect()
9501}
9502
9503/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9504/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9505/// this is data collection, not the fix-specific completion contract in
9506/// [`Runner::continue_fix_report`], and applies regardless of which node
9507/// asked.
9508///
9509/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9510/// evidence from; `Failed` does not, and correctly contributes nothing — a
9511/// timeout or crash is not itself evidence about a command the seat may have
9512/// started.
9513fn record_jobs(
9514    state: &mut RunState,
9515    node: &str,
9516    round: Option<usize>,
9517    seat: &str,
9518    out: &AgentOutcome,
9519) {
9520    let commands: &[agent::CommandEvidence] = match out {
9521        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9522        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9523    };
9524    let checked_at = Timestamp::now();
9525    for c in commands {
9526        state.jobs.push(JobRecord {
9527            node: node.to_owned(),
9528            round,
9529            seat: seat.to_owned(),
9530            id: c.id.clone(),
9531            description: c.description.clone(),
9532            checked_at,
9533            status: match c.exit_code {
9534                Some(0) => JobStatus::Completed,
9535                Some(_) => JobStatus::Failed,
9536                None => JobStatus::Unknown,
9537            },
9538            exit_code: c.exit_code,
9539            result_summary: c.result_summary.clone(),
9540            source: c.source.clone(),
9541        });
9542    }
9543}
9544
9545/// Is a review round clean, given how many reviewer seats answered against
9546/// how many the round expected?
9547///
9548/// A seat that never answered (timeout, crash, unparsable output) is not a
9549/// seat that read the patch and found nothing — treating it as such is
9550/// exactly the bug this function exists to close. Under the default `block`
9551/// policy a missing seat can never be clean; `warn` still requires the seats
9552/// that *did* answer to have found nothing blocking and verification to be
9553/// green.
9554///
9555/// `quota_missing` narrows that `block` default for exactly one cause of
9556/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9557/// a session limit lifts by the very next round buys nothing — the seat is
9558/// asked again with the same quota — so once every missing seat is accounted
9559/// for by a quota loss (and at least one seat *did* answer, so a decision has
9560/// something to rest on) the round is decided on the panel that could answer,
9561/// same as `warn` would. A panel that lost every seat to quota is not
9562/// decided here: `answered == 0` falls through to the existing `block`
9563/// fallback so a fully collapsed panel still waits rather than landing on no
9564/// review at all.
9565fn round_is_clean(
9566    blocking: usize,
9567    e2e_ok: bool,
9568    answered: usize,
9569    expected: usize,
9570    quota_missing: usize,
9571    policy: IncompleteReviewPolicy,
9572) -> bool {
9573    if blocking != 0 || !e2e_ok {
9574        return false;
9575    }
9576    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9577        return true;
9578    }
9579    answered > 0 && expected - answered <= quota_missing
9580}
9581
9582/// The review loop's own conclusion, derived entirely from its persisted
9583/// round records and the round budget that produced them — never from
9584/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9585/// recomputes the identical answer regardless of what an earlier node in the
9586/// same walk, or a previous walk, did to `status`.
9587///
9588/// `None` while more rounds remain to try, including when review never ran
9589/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9590/// gone clean, or the budget is spent, or the tree has stopped moving (see
9591/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9592///
9593/// - An incomplete panel that raised nothing is missing input, not a
9594///   verified tree — never a hand-off candidate, whatever verification said
9595///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9596/// - Otherwise, green e2e on the last round hands off (see
9597///   [`Runner::stop_reviewing`]); red e2e blocks.
9598///
9599/// A last round whose own verification is still `ResourceBlocked` — magi
9600/// itself never got a command to run, not evidence the patch is broken —
9601/// is neither: this returns `None` for it too, the same as "more rounds
9602/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9603/// handling of that shape) instead of this cheap recomputation guessing a
9604/// verdict a real attempt never produced.
9605pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9606    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9607        return Some(RunStatus::Gating);
9608    }
9609    let last = reviews.last()?;
9610    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9611    if reviews.len() < max_rounds && !stagnant {
9612        return None;
9613    }
9614    if last.incomplete() && last.blocking == 0 {
9615        return Some(RunStatus::Blocked);
9616    }
9617    if last.e2e_status() == E2eStatus::ResourceBlocked {
9618        return None;
9619    }
9620    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9621        RunStatus::Gating
9622    } else {
9623        RunStatus::Blocked
9624    })
9625}
9626
9627/// How long a re-ask may take, given the budget the first attempt had.
9628///
9629/// A `nudged` retry is a request to restate an answer the seat has already
9630/// worked out: it carries no new work, so it does not deserve the original
9631/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9632/// 133 seconds while a third sat for over ten minutes on a resumed session
9633/// holding 410 KB of prior output - and because the retry had inherited the
9634/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9635/// judging round whose other seats were long finished.
9636///
9637/// A quarter of the budget, with a floor so that a deliberately short timeout
9638/// does not collapse to nothing. A retry that re-sends the whole prompt
9639/// (because the seat kept no context) is the original job again, and keeps the
9640/// original budget.
9641pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9642    if nudged {
9643        (full / 4).max(Duration::from_secs(120)).min(full)
9644    } else {
9645        full
9646    }
9647}
9648
9649/// Run a wave and parse each reply, re-asking the seats whose reply was
9650/// unusable.
9651///
9652/// The re-ask is a nudge rather than the whole prompt again when the seat still
9653/// holds its conversation, which is the difference between a cheap retry and
9654/// paying for the entire candidate set twice.
9655///
9656/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9657/// successor in `roster` is handed to it instead of being re-asked: the
9658/// handover is the retry. A seat with no successor left (a single-agent
9659/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9660/// the asks to one seat in one node number at most
9661/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9662/// is asked once (a dropped stream is nudged first), and only the last agent
9663/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9664/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9665/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9666/// other failure stops the chain when the previous agent failed the same way
9667/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9668/// [`has_context`] is false and the job's own full prompt and full budget are
9669/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9670/// intermediate ones are not losses) and is returned as a failure like any
9671/// other absent seat — the caller decides whether the panel still has a
9672/// quorum. An empty `roster` disables handover: failures are nudged as they
9673/// always were, and a quota is simply lost. A reply that fails to parse or
9674/// validate is the prompt's doing and is only ever nudged, never handed over.
9675///
9676/// The returned [`SeatState`] names the agent that answered (or tried last).
9677#[allow(clippy::too_many_arguments)]
9678async fn ask_json_wave<T>(
9679    jobs: Vec<SeatJob>,
9680    sem: Arc<Semaphore>,
9681    retries: usize,
9682    roster: &[AgentSpec],
9683    ctx: &WaveCtx<'_>,
9684    losses: &mut Vec<QuotaLoss>,
9685    state: &mut RunState,
9686    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9687) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9688where
9689    T: serde::de::DeserializeOwned + Send + 'static,
9690{
9691    ask_wave_with(
9692        jobs,
9693        sem,
9694        retries,
9695        roster,
9696        ctx,
9697        losses,
9698        state,
9699        &|text: &str| {
9700            let v = verdict::extract_json::<T>(text)?;
9701            validate(&v)?;
9702            Ok(v)
9703        },
9704    )
9705    .await
9706}
9707
9708/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9709/// a node whose answer is prose (deliberation) shares the same handover,
9710/// failure classification, quota bookkeeping and bounds instead of a copy.
9711///
9712/// A seat handed to another roster agent is sent the job's `handover` prompt
9713/// (when it has one) rather than `prompt`: the new agent has no session, so a
9714/// resume-style prompt would be incomplete. That holds for the handover ask
9715/// and for every nudge to that agent whose `has_context` is false.
9716#[allow(clippy::too_many_arguments)]
9717async fn ask_wave_with<T>(
9718    jobs: Vec<SeatJob>,
9719    sem: Arc<Semaphore>,
9720    retries: usize,
9721    roster: &[AgentSpec],
9722    ctx: &WaveCtx<'_>,
9723    losses: &mut Vec<QuotaLoss>,
9724    state: &mut RunState,
9725    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9726) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9727where
9728    T: Send + 'static,
9729{
9730    let n = jobs.len();
9731    let originals: Vec<SeatJob> = jobs;
9732    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9733    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9734    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9735    // answer, N once it has gone through N nudges. Read back once this
9736    // returns, so a caller building a history record (`ReviewRecord`) can
9737    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9738    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9739    // field's own doc.
9740    let mut nudges: Vec<usize> = vec![0; n];
9741    // The agent now occupying each seat, the ids it has already been through,
9742    // where in the roster the walk began, the class of the last failure, and
9743    // the stem word of a handover not yet asked (full prompt, full budget).
9744    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9745    let mut tried: Vec<BTreeSet<String>> = specs
9746        .iter()
9747        .map(|s| BTreeSet::from([s.id.clone()]))
9748        .collect();
9749    let starts: Vec<usize> = specs
9750        .iter()
9751        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9752        .collect();
9753    let carry = ctx.carry_seats && !roster.is_empty();
9754    // Carried across rounds: ids that failed the seat earlier, and how the
9755    // last failure went (so a repeat of it is not handed over again).
9756    let carried: Vec<BTreeSet<String>> = originals
9757        .iter()
9758        .map(|j| {
9759            state
9760                .seat_history
9761                .get(&j.seat.key)
9762                .filter(|_| carry)
9763                .map(|h| h.failed.clone())
9764                .unwrap_or_default()
9765        })
9766        .collect();
9767    let mut prev: Vec<Option<FailClass>> = originals
9768        .iter()
9769        .map(|j| {
9770            state
9771                .seat_history
9772                .get(&j.seat.key)
9773                .filter(|_| carry)
9774                .and_then(|h| h.last_fail.clone())
9775        })
9776        .collect();
9777    let next_agent =
9778        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9779            let others: BTreeSet<String> = specs
9780                .iter()
9781                .enumerate()
9782                .filter(|(j, _)| *j != i)
9783                .map(|(_, s)| s.id.clone())
9784                .collect();
9785            pick_successor(
9786                roster,
9787                starts[i],
9788                tried,
9789                carry.then(|| &carried[i]),
9790                &others,
9791            )
9792            .cloned()
9793        };
9794    let mut fresh: Vec<Option<String>> = vec![None; n];
9795    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9796    let mut pending: Vec<usize> = (0..n).collect();
9797
9798    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
9799    // asks: an agent with a successor is asked once and handed over, and only
9800    // a seat with no successor spends `retries` nudges on the same agent. This
9801    // only guarantees the loop's own termination whatever those say.
9802    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9803    for round in 0..max_rounds {
9804        if pending.is_empty() {
9805            break;
9806        }
9807        let mut batch = Vec::with_capacity(pending.len());
9808        let mut renudged: Vec<&str> = Vec::new();
9809        for &i in &pending {
9810            let src = &originals[i];
9811            // A seat now held by another agent than the job named has no
9812            // session of its own: it gets the full-context prompt whenever
9813            // it is asked in full (the handover ask, a nudge it cannot
9814            // resume).
9815            let full: &str = match &src.handover {
9816                Some(h) if specs[i].id != src.spec.id => h,
9817                _ => &src.prompt,
9818            };
9819            // The prompt and the budget are one decision: a nudge restates
9820            // finished work, a re-sent prompt redoes it.
9821            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9822                (
9823                    full.to_owned(),
9824                    src.timeout,
9825                    format!("{}-{word}-{}", src.stem, specs[i].id),
9826                )
9827            } else if nudges[i] == 0 {
9828                (full.to_owned(), src.timeout, src.stem.clone())
9829            } else {
9830                renudged.push(src.seat.key.as_str());
9831                let why = done[i]
9832                    .as_ref()
9833                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
9834                    .unwrap_or_else(|| "no parsable answer".to_owned());
9835                let nudge = prompt::nudge(&why);
9836                let nudged = has_context(&specs[i], &seats[i], src.sessions);
9837                let prompt = if nudged {
9838                    nudge
9839                } else {
9840                    format!("{full}\n\n---\n\n{nudge}")
9841                };
9842                (
9843                    prompt,
9844                    retry_budget(src.timeout, nudged),
9845                    format!("{}-retry{}", src.stem, nudges[i]),
9846                )
9847            };
9848            batch.push(SeatJob {
9849                spec: specs[i].clone(),
9850                seat: seats[i].clone(),
9851                cwd: src.cwd.clone(),
9852                prompt,
9853                timeout,
9854                allow_write: src.allow_write,
9855                sessions: src.sessions,
9856                artifacts: src.artifacts.clone(),
9857                stem,
9858                handover: None,
9859            });
9860        }
9861
9862        if !renudged.is_empty() {
9863            state.event(
9864                ctx.node,
9865                format!("retry {round}: re-asking {}", renudged.join(", ")),
9866            );
9867        }
9868        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9869        let mut still = Vec::new();
9870        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9871            seats[i] = seat;
9872            // No agent was asked: nothing to hand over, nudge, remember or
9873            // charge. The caller sees `resource_block` and stops the node.
9874            if let AgentOutcome::ResourceBlocked(why) = &out {
9875                done[i] = Some(Err(anyhow::anyhow!("{why}")));
9876                continue;
9877            }
9878            let class = FailClass::of(&out);
9879            // A dropped stream is nudged first (the conversation is still
9880            // there to pick up); only a seat whose nudges are spent hands over.
9881            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9882                && nudges[i] < retries
9883                && !roster.is_empty();
9884            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9885                let next = should_hand_over(prev[i].as_ref(), &cur)
9886                    .then(|| next_agent(i, &tried[i], &specs))
9887                    .flatten();
9888                if carry {
9889                    let h = state
9890                        .seat_history
9891                        .entry(originals[i].seat.key.clone())
9892                        .or_default();
9893                    h.failed.insert(specs[i].id.clone());
9894                    h.last_fail = Some(cur.clone());
9895                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9896                        h.last_ok = None;
9897                    }
9898                    // Saved before the next agent is asked, so a restart in
9899                    // between does not forget who failed.
9900                    if let Err(e) = state.save() {
9901                        tracing::warn!("could not persist a seat's failure history: {e:#}");
9902                    }
9903                }
9904                if let Some(next) = next {
9905                    record_handover(
9906                        state,
9907                        ctx.node,
9908                        &originals[i].seat.key,
9909                        &specs[i].id,
9910                        &next.id,
9911                        &cur,
9912                        &fail_reason(&out),
9913                    );
9914                    tried[i].insert(next.id.clone());
9915                    prev[i] = Some(cur.clone());
9916                    seats[i] =
9917                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9918                    specs[i] = next;
9919                    fresh[i] = Some(cur.stem_word().to_owned());
9920                    nudges[i] = 0;
9921                    done[i] = Some(Err(anyhow::anyhow!(
9922                        "handed over after: {}",
9923                        fail_reason(&out)
9924                    )));
9925                    still.push(i);
9926                    continue;
9927                }
9928            }
9929            if carry
9930                && !nudge_first
9931                && let Some(cur) = class.clone()
9932            {
9933                // The chain ended here (no successor, or a repeated failure).
9934                let h = state
9935                    .seat_history
9936                    .entry(originals[i].seat.key.clone())
9937                    .or_default();
9938                h.failed.insert(specs[i].id.clone());
9939                h.last_fail = Some(cur);
9940            }
9941            let parsed = match out {
9942                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9943                AgentOutcome::Quota(o) => {
9944                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9945                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9946                }
9947                // Not a parseable answer: the nudge loop re-asks it, which is
9948                // exactly what a dropped stream needs. Just don't hand its raw
9949                // error JSON to `extract_json`.
9950                AgentOutcome::Dropped(o) => {
9951                    let why = o
9952                        .dropped
9953                        .as_ref()
9954                        .map(|d| d.why.as_str())
9955                        .unwrap_or("the CLI ended the stream without delivering its answer");
9956                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9957                }
9958                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9959                    Err(anyhow::anyhow!(e))
9960                }
9961            };
9962            let quota = class == Some(FailClass::Quota);
9963            let failed = parsed.is_err();
9964            done[i] = Some(parsed);
9965            if carry && !failed {
9966                let h = state
9967                    .seat_history
9968                    .entry(originals[i].seat.key.clone())
9969                    .or_default();
9970                h.failed.remove(&specs[i].id);
9971                h.last_ok = Some(specs[i].id.clone());
9972                h.last_fail = None;
9973            }
9974            // Do not re-ask a rate-limited seat (quota) — a retry is known to
9975            // fail the same way; and never re-ask a seat that already parsed.
9976            // A failed agent that still has a successor is not re-asked
9977            // either: the handover was its remedy and has just been refused
9978            // (the chain stops on a repeated failure class). A seat with no
9979            // successor left (a single-agent roster, the roster's tail, or an
9980            // empty roster) keeps the same-agent nudge, bounded by `retries`.
9981            let agent_failure = class.is_some()
9982                && !nudge_first
9983                && !roster.is_empty()
9984                && next_agent(i, &tried[i], &specs).is_some();
9985            if failed && !quota && !agent_failure && nudges[i] < retries {
9986                nudges[i] += 1;
9987                still.push(i);
9988            }
9989        }
9990        pending = still;
9991    }
9992
9993    // One loss per seat whose chain ended on a quota: the intermediate ones
9994    // were absorbed by a handover and are not losses.
9995    for (i, q) in last_quota.into_iter().enumerate() {
9996        if let Some(reset) = q {
9997            losses.push(QuotaLoss {
9998                seat: originals[i].seat.key.clone(),
9999                node: ctx.node.to_owned(),
10000                at: Timestamp::now(),
10001                reset,
10002            });
10003        }
10004    }
10005
10006    seats
10007        .into_iter()
10008        .zip(done)
10009        .zip(nudges)
10010        .map(|((seat, res), attempts)| {
10011            (
10012                seat,
10013                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10014                attempts,
10015            )
10016        })
10017        .collect()
10018}
10019
10020/// Did the wave end without asking any agent (see [`block_wave`])?
10021fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10022    results
10023        .iter()
10024        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10025}
10026
10027/// Settle a wave that never spawned anything: every job answers
10028/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10029/// else about the seat (session, history, quota) is touched.
10030fn block_wave(
10031    jobs: Vec<SeatJob>,
10032    state: &mut RunState,
10033    node: &str,
10034    attempt: usize,
10035    why: &str,
10036) -> Vec<(usize, SeatState, AgentOutcome)> {
10037    state.resource_wait = None;
10038    state
10039        .active
10040        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10041    if let Err(e) = state.save() {
10042        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10043    }
10044    jobs.into_iter()
10045        .enumerate()
10046        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10047        .collect()
10048}
10049
10050/// Acquire the shared build cache's lease, waiting out contention within
10051/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10052/// unbounded wait is never acceptable).
10053///
10054/// A first, non-blocking check happens before ever waiting; if it finds the
10055/// lease busy, that fact is logged as a `verify` event *and* flushed with
10056/// [`RunState::save`] immediately — not only once the wait finally succeeds
10057/// or gives up — so a `magi show` run by a different process while this one
10058/// is still waiting reads a `run.json` that says so, rather than whatever it
10059/// looked like before the wait started. The same applies to the terminal
10060/// failure: logged and saved before this returns `Err`, so a caller that
10061/// could not get the lease at all still leaves a legible record of why.
10062async fn acquire_cache_lease(
10063    state: &mut RunState,
10064    cache_dir: &Path,
10065    owner: &crate::cache::Owner,
10066    budget: Duration,
10067    context: &str,
10068    cancel: &(dyn Fn() -> bool + Sync),
10069) -> Result<Option<crate::cache::Guard>> {
10070    let home = crate::run::home();
10071    let started = Instant::now();
10072    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10073        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10074        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10075        Err(e) => {
10076            state.event(
10077                "verify",
10078                format!("{context}: could not check the shared build cache: {e:#}"),
10079            );
10080            if let Err(e2) = state.save() {
10081                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10082            }
10083            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10084        }
10085    };
10086    state.event(
10087        "verify",
10088        format!(
10089            "{context}: waiting for the shared build cache at {} ({})",
10090            cache_dir.display(),
10091            busy.describe()
10092        ),
10093    );
10094    // Seats of this node are waiting, not running: say so, so the countdown a
10095    // reader sees only ever measures an agent that has been spawned.
10096    state.resource_wait = Some(crate::run::ResourceWait {
10097        node: context.to_owned(),
10098        kind: "shared build cache".to_owned(),
10099        holder: busy.describe(),
10100        since: Timestamp::now(),
10101        limit_secs: budget.as_secs(),
10102    });
10103    if let Err(e) = state.save() {
10104        tracing::warn!("could not persist a cache wait: {e:#}");
10105    }
10106    let remaining = budget.saturating_sub(started.elapsed());
10107    let waited = crate::cache::wait_for_unless(
10108        &home,
10109        cache_dir,
10110        owner,
10111        remaining,
10112        Duration::from_secs(5),
10113        cancel,
10114    )
10115    .await;
10116    state.resource_wait = None;
10117    match waited {
10118        Ok(g) => Ok(g),
10119        Err(e) => {
10120            state.event("verify", format!("{context}: {e:#}"));
10121            if let Err(e2) = state.save() {
10122                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10123            }
10124            Err(e)
10125        }
10126    }
10127}
10128
10129/// The lease state could not be read at all (unreadable lease file, a
10130/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10131/// known to be contended, so a wave degrades to running without a shared cache
10132/// instead of stopping on a hold that [`crate::triage`] could never release.
10133#[derive(Debug)]
10134struct CacheCheckError(String);
10135
10136impl std::fmt::Display for CacheCheckError {
10137    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10138        f.write_str(&self.0)
10139    }
10140}
10141
10142impl std::error::Error for CacheCheckError {}
10143
10144/// Run `body` — a verify command batch — while holding the shared build
10145/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10146/// never interleave with another borrower's build against the same
10147/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10148/// timeout, or a human's own `magi review`. See the `cache` module doc for
10149/// why this matters more than Cargo's own per-target locking covers — two
10150/// *different* worktrees building the same package name/version into one
10151/// cache directory is a staleness bug, not a lock contention one.
10152///
10153/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10154/// agent waves do. Repository scheduling also has its own bounded,
10155/// cancellable resource-wait policy. Contention must not consume the
10156/// verification budget: `body` receives the full `budget` after both leases
10157/// are acquired and the cache is fresh. Without a shared cache, only
10158/// repository coordination is acquired before handing over the full budget.
10159///
10160/// A lease that cannot be acquired within the resource wait is reported as a single
10161/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10162/// verification — the same shape a spawn failure already takes in
10163/// [`run_commands`], so a caller need not special-case it.
10164#[allow(clippy::too_many_arguments)]
10165async fn with_cache_lease<'s, F, Fut>(
10166    state: &'s mut RunState,
10167    cache_dir: Option<&Path>,
10168    node: &str,
10169    seat: &str,
10170    worktree: &Path,
10171    head: &str,
10172    budget: Duration,
10173    context: &str,
10174    wait: &crate::repo_lock::WaitPolicy,
10175    body: F,
10176) -> Result<(Vec<CommandOutcome>, bool)>
10177where
10178    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10179    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10180{
10181    // Match editing waves: cache first, repository second. In particular,
10182    // pre_gate must never hold the repository while waiting for a wave's cache.
10183    let started = Instant::now();
10184    let home = crate::run::home();
10185    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10186    let guard = if let Some(cache_dir) = cache_dir {
10187        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10188        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10189            wait.cancelled()
10190        })
10191        .await
10192        .and_then(|g| {
10193            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10194        }) {
10195            Ok(g) => Some(g),
10196            Err(e) => {
10197                return Ok((
10198                    vec![CommandOutcome {
10199                        command: "(waiting for the shared build cache)".into(),
10200                        code: None,
10201                        output_tail: e.to_string(),
10202                        duration_ms: started.elapsed().as_millis() as u64,
10203                        resource_blocked: true,
10204                    }],
10205                    false,
10206                ));
10207            }
10208        }
10209    } else {
10210        None
10211    };
10212    let _coordination = coordinate_operation(state, worktree, wait).await?;
10213    if let Some(cache_dir) = cache_dir {
10214        let identity = crate::cache::Identity::new(worktree, head);
10215        if let Err(e) = crate::cache::ensure_fresh(
10216            &home,
10217            cache_dir,
10218            &identity,
10219            &state.config.verify.cargo_manifests(),
10220        ) {
10221            state.event(
10222                "verify",
10223                format!(
10224                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10225                    worktree.display(),
10226                    short(head)
10227                ),
10228            );
10229            if let Some(guard) = guard {
10230                guard.release();
10231            }
10232            return Ok((
10233                vec![CommandOutcome {
10234                    command: "(confirming the shared build cache is fresh)".into(),
10235                    code: None,
10236                    output_tail: e.to_string(),
10237                    duration_ms: started.elapsed().as_millis() as u64,
10238                    resource_blocked: true,
10239                }],
10240                false,
10241            ));
10242        }
10243    }
10244    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10245    if !timed_out_pids.is_empty() {
10246        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10247    }
10248    // Drop repository coordination before releasing the cache, reversing the
10249    // acquisition order. A cancelled/error future also drops both guards.
10250    drop(_coordination);
10251    if let Some(guard) = guard {
10252        guard.release();
10253    }
10254    Ok((outcomes, retried))
10255}
10256
10257/// Poll `pids` — commands [`run_commands`] reports as still running when its
10258/// own timeout elapsed — until every one is confirmed gone, or
10259/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10260///
10261/// Real confirmation where confirmation is possible, not a substitute for
10262/// full process-tree observation: a grandchild the timed-out process spawned
10263/// and that survives independently of it is invisible to a pid check the
10264/// same way it always was, and continuing to observe and collect *that*
10265/// stays a different piece of work with its own owner. This only narrows a
10266/// fixed blind wait into an actual check of the pids this process does know
10267/// about.
10268async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10269    wait_for_pids_with(
10270        pids,
10271        crate::proc::pid_alive,
10272        LEASE_RELEASE_POLL,
10273        LEASE_RELEASE_MAX_WAIT,
10274    )
10275    .await;
10276}
10277
10278/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10279/// interval and ceiling supplied by the caller, so the polling *logic* -
10280/// returns as soon as every pid reports dead, gives up at the ceiling
10281/// otherwise - is testable on millisecond durations without asking the real
10282/// OS about a pid at all.
10283async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10284    pids: &[u32],
10285    alive: F,
10286    poll: Duration,
10287    max_wait: Duration,
10288) {
10289    let deadline = Instant::now() + max_wait;
10290    loop {
10291        if pids.iter().all(|&pid| !alive(pid)) {
10292            return;
10293        }
10294        if Instant::now() >= deadline {
10295            return;
10296        }
10297        tokio::time::sleep(poll).await;
10298    }
10299}
10300
10301/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10302/// admission that it could not even get a verify command to run, as opposed
10303/// to evidence the command actually produced? A caller that would otherwise
10304/// read a resource-blocked outcome as a red command must check this first:
10305/// see [`Runner::gate`], which retries rather than records `Blocked` when
10306/// this is true.
10307fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10308    outcomes.iter().any(|o| o.resource_blocked)
10309}
10310
10311/// What [`Runner::gate_fix_round`] decided.
10312pub(crate) enum GateFix {
10313    /// The tree changed and `verify.e2e` is still green: run the gate again.
10314    Retry,
10315    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10316    /// last failure stands and the run ends blocked.
10317    Stop,
10318    /// `verify.e2e` could not run after the fix (magi's own contention):
10319    /// decide nothing now, a later reentry retries.
10320    Defer,
10321}
10322
10323/// Is every red command in `outcomes` an ordinary failure the code could
10324/// explain: it ran, exited non-zero, and said something?
10325///
10326/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10327/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10328/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10329/// what the command is stays the gate's business.
10330fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10331    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10332    red.peek().is_some()
10333        && red.all(|o| {
10334            !o.resource_blocked
10335                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10336                && !o.output_tail.trim().is_empty()
10337        })
10338}
10339
10340/// Describe one verify command's outcome for the event log, distinguishing a
10341/// build/link failure — the toolchain never produced a binary to run — from
10342/// an actual test failure, since only the latter is a verdict on the patch.
10343fn e2e_outcome_label(o: &CommandOutcome) -> String {
10344    if o.ok() {
10345        return "pass".to_owned();
10346    }
10347    let reason = if o.build_failed() {
10348        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10349    } else {
10350        format!("FAIL ({:?})", o.code)
10351    };
10352    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10353}
10354
10355/// Run `verify.e2e`, retrying once if the first attempt could not build or
10356/// link — a build/link failure is frequently a race against a shared
10357/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10358/// `verify` event per command, tagged with `context` (normally `"round N"`)
10359/// so the two call sites that need this — the ordinary per-round leg in
10360/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10361/// it will ever call a round green — read identically in the event log.
10362async fn run_e2e_with_retry(
10363    state: &mut RunState,
10364    shell: &[String],
10365    commands: &[String],
10366    worktree: &Path,
10367    timeout: Duration,
10368    context: &str,
10369) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10370    let (mut e2e, mut timed_out_pids) = run_commands(
10371        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10372    )
10373    .await;
10374    for o in &e2e {
10375        state.event(
10376            "verify",
10377            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10378        );
10379    }
10380    // A build/link failure is not a verdict on the patch — it is frequently a
10381    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10382    // one retry before letting a red like that decide the round.
10383    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10384    if verify_retried {
10385        state.event(
10386            "verify",
10387            format!(
10388                "{context}: verify could not build/link, not a test result — retrying once \
10389                 before concluding"
10390            ),
10391        );
10392        let retried = run_commands(
10393            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10394        )
10395        .await;
10396        e2e = retried.0;
10397        // Both attempts' timeouts matter, not just the last one: the first
10398        // attempt's descendants may still be alive alongside the retry's.
10399        timed_out_pids.extend(retried.1);
10400        for o in &e2e {
10401            state.event(
10402                "verify",
10403                format!(
10404                    "{context}: retry `{}` -> {}",
10405                    o.command,
10406                    e2e_outcome_label(o)
10407                ),
10408            );
10409        }
10410    }
10411    (e2e, verify_retried, timed_out_pids)
10412}
10413
10414/// Run configured shell commands in `cwd`, in order. The second element is
10415/// the pid of every command that hit `timeout` and was still running when
10416/// this stopped waiting on it (best-effort: `None` when the platform did not
10417/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10418/// that releases a shared resource afterward needs to know.
10419///
10420/// Records `task` into [`RunState::active`] at every command boundary
10421/// (`RunState::task_command`) and clears it once the whole list has run
10422/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10423/// for minutes with no seat and no output of its own to show for it (see
10424/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10425/// apart from "not yet run" without this), and this is the only place that
10426/// knows which command is running right now and how many are left. Three
10427/// saves per command — start, not per second — matching the same "only at a
10428/// boundary" rule [`wave`] already follows for seats.
10429#[allow(clippy::too_many_arguments)]
10430async fn run_commands(
10431    state: &mut RunState,
10432    node: &str,
10433    task: &str,
10434    attempt: usize,
10435    shell: &[String],
10436    commands: &[String],
10437    cwd: &Path,
10438    timeout: Duration,
10439) -> (Vec<CommandOutcome>, Vec<u32>) {
10440    if commands.is_empty() {
10441        // Nothing to mark as running and nothing to clear — an empty list
10442        // means "not configured", and touching `active` (or the disk) over
10443        // that would be a write for every round of a repo with no
10444        // `verify.e2e` / `verify.gate` commands at all.
10445        return (Vec::new(), Vec::new());
10446    }
10447    let mut out = Vec::new();
10448    let mut timed_out_pids = Vec::new();
10449    let total = commands.len();
10450    for (idx, command) in commands.iter().enumerate() {
10451        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10452        if let Err(e) = state.save() {
10453            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10454        }
10455        let started = Instant::now();
10456        let mut cmd = tokio::process::Command::new(&shell[0]);
10457        cmd.quiet();
10458        cmd.args(&shell[1..])
10459            .arg(command)
10460            .current_dir(cwd)
10461            .stdin(std::process::Stdio::null())
10462            .stdout(std::process::Stdio::piped())
10463            .stderr(std::process::Stdio::piped())
10464            .kill_on_drop(true);
10465        let mut outputs = Vec::new();
10466        let protected = (|| -> Result<()> {
10467            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10468                outputs.push(guard);
10469            }
10470            if let Some(cache) = state.config.cache_dir() {
10471                outputs.push(crate::retention::OutputLease::cache(
10472                    &crate::run::home(),
10473                    &cache,
10474                    false,
10475                )?);
10476            }
10477            Ok(())
10478        })();
10479        if let Err(e) = protected {
10480            out.push(CommandOutcome {
10481                command: command.clone(),
10482                code: None,
10483                output_tail: format!("output ownership unavailable: {e:#}"),
10484                duration_ms: 0,
10485                resource_blocked: true,
10486            });
10487            continue;
10488        }
10489        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10490        let (code, body) = match spawned {
10491            Ok(mut running) => {
10492                let pid = running.child().id();
10493                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10494                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10495                let mut paths = vec![cwd.to_owned()];
10496                if let Some(cache) = state.config.cache_dir() {
10497                    paths.push(cache);
10498                }
10499                let mut pressure = None;
10500                let status = tokio::select! {
10501                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10502                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10503                        pressure = Some(reason); None
10504                    }
10505                };
10506                let (code, failure) = match status {
10507                    Some(Ok(Ok(status))) => (
10508                        status.code(),
10509                        status
10510                            .code()
10511                            .is_none()
10512                            .then(|| format!("command terminated: {status}")),
10513                    ),
10514                    Some(Ok(Err(error))) => {
10515                        (None, Some(format!("failed waiting for command: {error}")))
10516                    }
10517                    Some(Err(_)) => {
10518                        // Leased cleanup confirms process-tree death itself.
10519                        // Unleased commands still need the caller's PID check
10520                        // after kill-on-drop, before releasing cache ownership.
10521                        if !running.coordinated()
10522                            && let Some(pid) = pid
10523                        {
10524                            timed_out_pids.push(pid);
10525                        }
10526                        (
10527                            None,
10528                            Some(format!("timed out after {}s", timeout.as_secs())),
10529                        )
10530                    }
10531                    None => (None, None),
10532                };
10533                running.finish().await;
10534                if let Some(reason) = pressure {
10535                    state.resource_block = Some(crate::run::ResourceBlock {
10536                        node: node.to_owned(),
10537                        kind: "disk pressure".to_owned(),
10538                        detail: reason.clone(),
10539                        at: Timestamp::now(),
10540                    });
10541                    out.push(CommandOutcome {
10542                        command: command.clone(),
10543                        code: None,
10544                        output_tail: reason,
10545                        duration_ms: started.elapsed().as_millis() as u64,
10546                        resource_blocked: true,
10547                    });
10548                    break;
10549                }
10550                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10551                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10552                if let Some(failure) = failure {
10553                    body.push('\n');
10554                    body.push_str(&failure);
10555                }
10556                (code, body)
10557            }
10558            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10559        };
10560        out.push(CommandOutcome {
10561            command: command.clone(),
10562            code,
10563            output_tail: tail(&body, OUTPUT_TAIL),
10564            duration_ms: started.elapsed().as_millis() as u64,
10565            resource_blocked: false,
10566        });
10567    }
10568    state.task_finished(task);
10569    if let Err(e) = state.save() {
10570        tracing::warn!("could not persist the end of {task}: {e:#}");
10571    }
10572    (out, timed_out_pids)
10573}
10574
10575/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10576/// section (`report::run`) and in the `merge` event this node records — for
10577/// the operator to run by hand.
10578///
10579/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10580/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10581/// merge commits", or "require linear history") rejects the push a `--no-ff`
10582/// merge would produce, which is exactly the guidance this function replaces.
10583/// `message`'s first line becomes the squash commit's subject, matching the
10584/// note `report::run` prints alongside this command — see that function for
10585/// why an explicit subject is not optional there.
10586fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10587    let repo = repo.display();
10588    match style {
10589        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10590        MergeStyle::Squash => {
10591            // The subject sits inside double quotes, and a title an agent
10592            // wrote may carry the characters that break out of them.
10593            let subject = message
10594                .lines()
10595                .next()
10596                .unwrap_or(branch)
10597                .replace(['\\', '"', '$', '`'], "");
10598            format!(
10599                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10600            )
10601        }
10602        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10603    }
10604}
10605
10606/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10607/// under the hood, rejects a `title` over 256 characters and the whole
10608/// command fails — no PR at all, for a run whose body was otherwise fine
10609/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10610/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10611/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10612/// this bound. It is a margin, not a guarantee — a title packed
10613/// with multi-unit characters could still in principle land close to the
10614/// edge, but a real task title's occasional emoji or accented letter fits
10615/// comfortably inside it.
10616const PR_TITLE_MAX: usize = 240;
10617
10618/// A pull request title from the opening line of `text`, or `None` when that
10619/// line is not English (GitHub text is) or has no letters.
10620///
10621/// A line within `max` is kept as is. A longer one is cut at the end of its
10622/// first sentence when that falls inside `max`, else at a word boundary with a
10623/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10624/// make every merely-truncated title look non-English). The language check
10625/// runs on the kept text before any mark is added, so only what GitHub will
10626/// show is judged: an English opening followed by non-ASCII far past the cut
10627/// still passes.
10628fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10629    let line = queue::first_line(text)?;
10630    let chars: Vec<char> = line.chars().collect();
10631    let (kept, mark) = if chars.len() <= max {
10632        (line.to_owned(), "")
10633    } else if let Some(end) = sentence_end(&chars, max) {
10634        (chars[..end].iter().collect::<String>(), "")
10635    } else {
10636        let room = max.saturating_sub(3);
10637        // Cut at the last space inside the room; when the char just past the
10638        // room is a space the room already ends on a word.
10639        let cut = if chars[room].is_whitespace() {
10640            room
10641        } else {
10642            chars[..room]
10643                .iter()
10644                .rposition(|c| c.is_whitespace())
10645                .unwrap_or(room)
10646        };
10647        let head: String = chars[..cut].iter().collect();
10648        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10649        (head.to_owned(), "...")
10650    };
10651    if kept.is_empty() {
10652        return None;
10653    }
10654    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10655        return None;
10656    }
10657    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10658        return None;
10659    }
10660    Some(format!("{kept}{mark}"))
10661}
10662
10663/// The char length of the first sentence of `chars` when it ends within `max`
10664/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10665/// abbreviations so `e.g. foo` does not end a title early.
10666fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10667    const MIN: usize = 20;
10668    for i in MIN..max.min(chars.len()) {
10669        if !matches!(chars[i], '.' | '!' | '?') {
10670            continue;
10671        }
10672        let Some(&next) = chars.get(i + 1) else {
10673            continue;
10674        };
10675        if !next.is_whitespace() {
10676            continue;
10677        }
10678        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10679        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10680            continue;
10681        }
10682        let word: String = chars[..i]
10683            .iter()
10684            .rev()
10685            .take_while(|c| !c.is_whitespace())
10686            .collect::<Vec<_>>()
10687            .into_iter()
10688            .rev()
10689            .collect();
10690        let word = word.to_ascii_lowercase();
10691        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10692            continue;
10693        }
10694        let end = chars[..i]
10695            .iter()
10696            .rposition(|c| !c.is_whitespace())
10697            .map_or(i, |p| p + 1);
10698        return Some(end);
10699    }
10700    None
10701}
10702
10703/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10704/// change: a title and a body describing what was *implemented*, not the task
10705/// that asked for it. A task reads as a request; a reader of the merged
10706/// history wants the change.
10707struct PrMessage {
10708    title: String,
10709    body: String,
10710}
10711
10712impl PrMessage {
10713    /// Title, blank line, body. The first line is the squash/merge commit
10714    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10715    /// has to stay one sensible line.
10716    fn commit_message(&self) -> String {
10717        format!("{}\n\n{}", self.title, self.body)
10718    }
10719}
10720
10721/// The text after a leading `TITLE:` (any case) on `line`.
10722fn title_marker(line: &str) -> Option<&str> {
10723    let line = line.trim();
10724    let head = line.get(..6)?;
10725    head.eq_ignore_ascii_case("title:")
10726        .then(|| line[6..].trim())
10727}
10728
10729/// The implementer's own one-line title: the `TITLE:` line the implement
10730/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10731/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10732/// source, and a title that says as much is refused here too.
10733fn summary_title(summary: &str) -> Option<String> {
10734    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10735    let raw = title_marker(first)?;
10736    if raw.is_empty() {
10737        return None;
10738    }
10739    let title = queue::title_from(raw, PR_TITLE_MAX);
10740    let lower = title.to_ascii_lowercase();
10741    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10742        return None;
10743    }
10744    Some(title)
10745}
10746
10747/// How `open_review`'s instruction begins; see [`landing_title`].
10748const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10749
10750/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10751pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10752
10753/// Does this run review work that already existed, rather than implement a
10754/// task? Runs recorded before `reviewed_commits` existed carry only the
10755/// candidate marker.
10756fn is_review_run(state: &RunState) -> bool {
10757    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10758}
10759
10760/// The title of a review-only run: the subject of the oldest commit under
10761/// review. Later commits are usually fixups, and `instruction` is the review
10762/// prompt, which says nothing about the change. GitHub text is English, so a
10763/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10764fn review_title(state: &RunState) -> Option<String> {
10765    english_subject(
10766        state.reviewed_commits.as_ref()?.first()?,
10767        state.config.graph.github_english(),
10768    )
10769}
10770
10771/// `raw` as a pull request title, or `None` when it is blank, not English
10772/// (GitHub text is), or one of magi's own candidate commit subjects.
10773fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10774    let raw = raw.trim();
10775    if raw.is_empty() {
10776        return None;
10777    }
10778    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10779        return None;
10780    }
10781    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10782        return None;
10783    }
10784    let title = queue::title_from(raw, PR_TITLE_MAX);
10785    let lower = title.to_ascii_lowercase();
10786    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10787        return None;
10788    }
10789    Some(title)
10790}
10791
10792/// What a review-only run's branch says about itself, read at the moment the
10793/// pull request is opened.
10794#[derive(Debug, Clone, PartialEq, Eq)]
10795struct BranchFacts {
10796    /// `(subject, body)` of each commit, oldest first.
10797    commits: Vec<(String, String)>,
10798    /// Trimmed `git diff --stat`.
10799    stat: String,
10800}
10801
10802/// Longest diff stat shown: this many file lines plus the summary line.
10803const STAT_FILE_LINES: usize = 25;
10804/// Cap on the commit list, well inside GitHub's 65536-character body limit.
10805const COMMITS_MAX_CHARS: usize = 20_000;
10806
10807/// Read the commits and diff stat of `base..branch`. `None` when git cannot
10808/// say or finds nothing, so the caller falls back to what the run recorded.
10809async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10810    let commits = git::commit_log(repo, base, branch).await.ok()?;
10811    if commits.is_empty() {
10812        return None;
10813    }
10814    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10815    let lines: Vec<&str> = stat.lines().collect();
10816    let stat = if lines.len() > STAT_FILE_LINES + 1 {
10817        let omitted = lines.len() - 1 - STAT_FILE_LINES;
10818        let more = format!(" ... {omitted} more file(s)");
10819        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10820        kept.push(&more);
10821        kept.push(lines[lines.len() - 1]);
10822        kept.join("\n")
10823    } else {
10824        lines.join("\n")
10825    };
10826    Some(BranchFacts { commits, stat })
10827}
10828
10829/// Defang what would break the surrounding markdown: a closing `</details>`
10830/// and a code fence.
10831fn markdown_safe(text: &str) -> String {
10832    text.replace("</details>", "&lt;/details&gt;")
10833        .replace("\x60\x60\x60", "~~~")
10834}
10835
10836fn neutral_title(state: &RunState, winner: char) -> String {
10837    format!(
10838        "chore: land candidate {} of run {}",
10839        winner.to_ascii_uppercase(),
10840        state.id
10841    )
10842}
10843
10844/// Reuse the graph's bounded fixer without starting another graph or driver.
10845/// Copy the working state back even on errors so spent rounds remain visible.
10846pub(crate) async fn repair_landing_gate(
10847    state: &mut RunState,
10848    winner: &Candidate,
10849    outcomes: &[CommandOutcome],
10850) -> Result<GateFix> {
10851    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10852        return Ok(GateFix::Stop);
10853    }
10854    let mut runner = Runner {
10855        roles: state.config.resolve_roles()?,
10856        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10857        state: state.clone(),
10858        pause: Pause::new(),
10859        interrupt: Pause::new(),
10860    };
10861    let result = runner.gate_fix_round(winner, outcomes).await;
10862    *state = runner.state;
10863    result
10864}
10865
10866/// Recheck a repair's e2e requirement across failures, resource deferrals and
10867/// process interruptions. A successful result only applies to its exact SHA.
10868pub(crate) async fn check_gate_repair_e2e(
10869    state: &mut RunState,
10870    winner: &Candidate,
10871    wait: &crate::repo_lock::WaitPolicy,
10872) -> Result<GateFix> {
10873    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10874        return Ok(GateFix::Retry);
10875    }
10876    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10877    if state
10878        .gate_repair_e2e
10879        .as_ref()
10880        .is_some_and(|r| r.head == head && r.passed)
10881    {
10882        return Ok(GateFix::Retry);
10883    }
10884    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10885        head: head.clone(),
10886        passed: false,
10887        outcomes: Vec::new(),
10888    });
10889    state.save()?;
10890    let commands = state.config.verify.e2e.clone();
10891    let outcomes = if commands.is_empty() {
10892        Vec::new()
10893    } else {
10894        let shell = state.config.shell();
10895        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10896        let cache_dir = state.config.cache_dir();
10897        let (outcomes, _) = with_cache_lease(
10898            state,
10899            cache_dir.as_deref(),
10900            "e2e",
10901            "e2e",
10902            &winner.worktree,
10903            &head,
10904            timeout,
10905            "gate repair e2e",
10906            wait,
10907            |state, budget| async move {
10908                run_e2e_with_retry(
10909                    state,
10910                    &shell,
10911                    &commands,
10912                    &winner.worktree,
10913                    budget,
10914                    "gate repair e2e",
10915                )
10916                .await
10917            },
10918        )
10919        .await?;
10920        outcomes
10921    };
10922    let deferred = verify_inconclusive(&outcomes);
10923    let passed = outcomes.iter().all(CommandOutcome::ok)
10924        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10925    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10926        head,
10927        passed,
10928        outcomes,
10929    });
10930    state.save()?;
10931    if deferred {
10932        Ok(GateFix::Defer)
10933    } else if !passed {
10934        state.event(
10935            "gate",
10936            "verify.e2e failed after gate repair; nothing published",
10937        );
10938        Ok(GateFix::Stop)
10939    } else {
10940        Ok(GateFix::Retry)
10941    }
10942}
10943
10944/// Run the same configured gate for a landing head and a fresh graph.
10945pub(crate) async fn run_landing_gate(
10946    state: &mut RunState,
10947    winner: &Candidate,
10948    wait: &crate::repo_lock::WaitPolicy,
10949) -> Result<Vec<CommandOutcome>> {
10950    let shell = state.config.shell();
10951    let gate_commands = state.config.verify.gate.clone();
10952    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10953    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10954        Vec::new()
10955    } else {
10956        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10957        let cache_dir = state.config.cache_dir();
10958        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10959        let (outcomes, _) = with_cache_lease(
10960            state,
10961            cache_dir.as_deref(),
10962            "gate",
10963            "gate",
10964            &winner.worktree,
10965            &head,
10966            timeout,
10967            "final gate",
10968            wait,
10969            |state, budget| {
10970                let shell = shell.clone();
10971                let gate_commands = gate_commands.clone();
10972                let winner = winner.clone();
10973                async move {
10974                    Runner::run_pre_gate(state, &winner).await;
10975                    let (outcomes, timed_out_pids) = run_commands(
10976                        state,
10977                        "gate",
10978                        "gate",
10979                        0,
10980                        &shell,
10981                        &gate_commands,
10982                        &winner.worktree,
10983                        budget,
10984                    )
10985                    .await;
10986                    (outcomes, false, timed_out_pids)
10987                }
10988            },
10989        )
10990        .await?;
10991        outcomes
10992    };
10993    if outcomes.is_empty() {
10994        // Nothing configured to check — distinct from every other
10995        // silence in this run's event log, since an empty `gate` alone
10996        // no longer says whether the gate ran at all (see
10997        // `RunState::gate_ran`'s own doc).
10998        state.event("gate", "no gate commands configured; not tested");
10999    }
11000    for o in &outcomes {
11001        state.event(
11002            "gate",
11003            format!(
11004                "`{}` -> {}",
11005                o.command,
11006                if o.ok() {
11007                    "pass".to_owned()
11008                } else {
11009                    format!(
11010                        "FAIL ({:?})\n{}",
11011                        o.code,
11012                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11013                    )
11014                }
11015            ),
11016        );
11017    }
11018    Ok(outcomes)
11019}
11020
11021/// The pull request title to hand to `land::merge_subject`. A review-only run
11022/// opened by an earlier build titled its pull request with the review prompt;
11023/// that title is dropped (empty, so the fallback applies) rather than landed.
11024/// Any other title, including an operator's rename, passes through untouched,
11025/// and so does every title of a run that implements a task.
11026pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11027    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11028        ""
11029    } else {
11030        pr_title
11031    }
11032}
11033
11034/// What the squash subject falls back to when the pull request title is empty
11035/// or candidate-shaped: for a review-only run the derived title, never the
11036/// review prompt held in `instruction`.
11037pub fn landing_subject_source(state: &RunState) -> String {
11038    if is_review_run(state) {
11039        let winner = state.candidates.first().map_or('A', |c| c.label);
11040        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11041    }
11042    state.instruction.clone()
11043}
11044
11045/// `summary` without its `TITLE:` line, which the pull request title already
11046/// carries.
11047fn summary_without_title(summary: &str) -> String {
11048    let mut lines = summary.trim().lines().peekable();
11049    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11050        lines.next();
11051    }
11052    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11053}
11054
11055/// The pull request title and body for the winning candidate.
11056///
11057/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11058/// the task's own opening line via [`queue::title_from`] when there is none.
11059/// `state.instruction` can open with blank lines (`task_text` only rejects a
11060/// body that is blank *entirely*), which `title_from` skips.
11061///
11062/// Publishing uses an independent description generated from the final diff
11063/// and structured evidence. Internal summaries and reports remain local.
11064/// Non-publishing previews may display the original task and candidate summary.
11065/// The footer retains the run and candidate provenance.
11066#[cfg(test)]
11067fn pr_message(state: &RunState, winner: char) -> PrMessage {
11068    pr_message_with(state, winner, None)
11069}
11070
11071/// [`pr_message`] with what the branch of a review-only run says about itself.
11072/// `facts` is ignored for a run that implements a task.
11073#[cfg(test)]
11074fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11075    let raw = pr_message_raw(
11076        state,
11077        winner,
11078        facts,
11079        crate::pr_outcome::FinalHead::Unreported,
11080        ChangeNote::NotAsked,
11081    );
11082    let id = crate::scrub::Identity::current();
11083    PrMessage {
11084        title: crate::scrub::scrub(&raw.title, &id),
11085        body: crate::scrub::scrub(&raw.body, &id),
11086    }
11087}
11088
11089/// What the public description may say about the change itself.
11090#[derive(Debug, Clone, Copy)]
11091enum ChangeNote<'a> {
11092    /// A local preview or non-publishing merge mode: no public generation needed.
11093    NotAsked,
11094    /// A summary written from the final diff.
11095    Summary(&'a crate::run::ChangeSummary),
11096    /// A summary was asked for and could not be produced; the text says why.
11097    Unavailable(&'a str),
11098}
11099
11100fn pr_message_raw(
11101    state: &RunState,
11102    winner: char,
11103    facts: Option<&BranchFacts>,
11104    head: crate::pr_outcome::FinalHead<'_>,
11105    change: ChangeNote<'_>,
11106) -> PrMessage {
11107    if !matches!(change, ChangeNote::NotAsked) {
11108        let (title, mut body) = match change {
11109            ChangeNote::Summary(c) => (
11110                c.title.clone(),
11111                format!("## Summary\n\n{}\n\n", c.body.trim()),
11112            ),
11113            ChangeNote::Unavailable(why) => (
11114                neutral_title(state, winner),
11115                format!(
11116                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11117                ),
11118            ),
11119            ChangeNote::NotAsked => unreachable!(),
11120        };
11121        body.push_str(&public_pr_evidence(state, head));
11122        body.push_str(&format!(
11123            "\n---\nmagi:run/{} magi:candidate-{}\n",
11124            state.id,
11125            winner.to_ascii_lowercase()
11126        ));
11127        return PrMessage { title, body };
11128    }
11129    pr_preview(state, winner, facts, head)
11130}
11131
11132/// Append public facts from the run records after every prose generation,
11133/// including posting-gate rewrites. The agent cannot drop this provenance.
11134fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11135    let mut body = crate::pr_outcome::review_rounds(state);
11136    body.push_str(&crate::pr_outcome::final_verification(state, head));
11137    let open = state.last_round_findings();
11138    if !open.is_empty() {
11139        body.push_str("## Open review findings\n\n");
11140        for f in open {
11141            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11142        }
11143        body.push('\n');
11144    }
11145    if let Some(c) = state.change_summary.as_ref()
11146        && head == crate::pr_outcome::FinalHead::At(&c.head)
11147    {
11148        body.push_str(&format!(
11149            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11150            c.base, c.head
11151        ));
11152    }
11153    body
11154}
11155
11156/// Local-only preview of the operator's task and candidate summary.
11157fn pr_preview(
11158    state: &RunState,
11159    winner: char,
11160    facts: Option<&BranchFacts>,
11161    head: crate::pr_outcome::FinalHead<'_>,
11162) -> PrMessage {
11163    let summary = state
11164        .candidates
11165        .iter()
11166        .find(|c| c.label == winner)
11167        .map(|c| c.summary.as_str())
11168        .unwrap_or_default();
11169    // The fallback is the operator's own words and may not be English; GitHub
11170    // text always is, so a non-English task gets a neutral title instead.
11171    let review = is_review_run(state);
11172    let title = if review {
11173        facts
11174            .and_then(|f| {
11175                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11176            })
11177            .or_else(|| review_title(state))
11178            .or_else(|| {
11179                state
11180                    .candidates
11181                    .iter()
11182                    .find(|c| c.label == winner)
11183                    .filter(|c| !c.branch.starts_with("magi/"))
11184                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11185            })
11186            .unwrap_or_else(|| neutral_title(state, winner))
11187    } else {
11188        summary_title(summary).unwrap_or_else(|| {
11189            english_title(
11190                &state.instruction,
11191                PR_TITLE_MAX,
11192                state.config.graph.github_english(),
11193            )
11194            .unwrap_or_else(|| neutral_title(state, winner))
11195        })
11196    };
11197
11198    let mut body = String::new();
11199    let what = summary_without_title(summary);
11200    if !what.is_empty() {
11201        body.push_str("## Summary\n\n");
11202        body.push_str(&what);
11203        body.push_str("\n\n");
11204    }
11205
11206    // Review history uses structured facts across all rounds, never reports.
11207    body.push_str(&crate::pr_outcome::review_rounds(state));
11208    body.push_str(&crate::pr_outcome::final_verification(state, head));
11209    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11210
11211    let open = state.open_findings();
11212    if !open.is_empty() {
11213        body.push_str("## Open review findings\n\n");
11214        for f in &open {
11215            body.push_str(&format!(
11216                "- `{}` [{:?}] {}\n",
11217                f.id, f.severity, "Context retained in local review records."
11218            ));
11219        }
11220        body.push('\n');
11221    }
11222
11223    if let Some(fix) = fix
11224        && !fix.rejected.is_empty()
11225    {
11226        body.push_str("## Declined by the fixer\n\n");
11227        for r in &fix.rejected {
11228            body.push_str(&format!(
11229                "- `{}`: {}\n",
11230                r.id, "Reason retained in local review records."
11231            ));
11232        }
11233        body.push('\n');
11234    }
11235
11236    if review {
11237        // The review prompt is not the task; list what the branch carries.
11238        body.push_str("## Commits under review\n\n");
11239        if let Some(facts) = facts {
11240            let mut left = COMMITS_MAX_CHARS;
11241            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11242                let mut entry = format!("- {}\n", markdown_safe(subject));
11243                for l in markdown_safe(text).lines() {
11244                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11245                }
11246                if left == 0 {
11247                    body.push_str(&format!(
11248                        "- ... {} more commit(s)\n",
11249                        facts.commits.len() - i
11250                    ));
11251                    break;
11252                }
11253                if entry.len() > left {
11254                    // Even the first commit is cut: one huge body must not
11255                    // push the whole description past GitHub's limit.
11256                    let mut end = left;
11257                    while !entry.is_char_boundary(end) {
11258                        end -= 1;
11259                    }
11260                    entry.truncate(end);
11261                    entry.push_str("\n  ... (truncated)\n");
11262                    left = 0;
11263                } else {
11264                    left -= entry.len();
11265                }
11266                body.push_str(&entry);
11267            }
11268            if !facts.stat.trim().is_empty() {
11269                body.push_str(&format!(
11270                    "\n## Diff stat\n\n```\n{}\n```\n",
11271                    markdown_safe(facts.stat.trim())
11272                ));
11273            }
11274        } else {
11275            match &state.reviewed_commits {
11276                Some(subjects) => {
11277                    for s in subjects {
11278                        body.push_str(&format!("- {}\n", s.trim()));
11279                    }
11280                }
11281                None => {
11282                    // An older run kept only the prompt, with the commit list
11283                    // after its first paragraph.
11284                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11285                    body.push_str(rest.trim());
11286                    body.push('\n');
11287                }
11288            }
11289        }
11290    } else {
11291        let task = state.instruction.trim();
11292        let task = if task.is_empty() {
11293            "(empty task)"
11294        } else {
11295            task
11296        };
11297        body.push_str(&format!(
11298            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11299            task.replace("</details>", "&lt;/details&gt;")
11300        ));
11301    }
11302
11303    body.push_str(&format!(
11304        "\n---\nmagi:run/{} magi:candidate-{}\n",
11305        state.id,
11306        winner.to_ascii_lowercase()
11307    ));
11308
11309    PrMessage { title, body }
11310}
11311
11312/// Source imports have already been prepared from structured inputs. Legacy
11313/// inferred references must never instruct a seat to reconstruct those imports.
11314fn seeded_instruction(state: &RunState) -> String {
11315    let mut inputs = Vec::new();
11316    if let Some(start) = &state.sources.start_from {
11317        inputs.push(format!(
11318            "Start from `{}` at `{}` (including ancestry).",
11319            start.token, start.sha
11320        ));
11321    }
11322    for pick in &state.sources.pick_commits {
11323        inputs.push(format!(
11324            "Pick `{}` at `{}` (single commit).",
11325            pick.token, pick.sha
11326        ));
11327    }
11328    if inputs.is_empty() {
11329        state.instruction.clone()
11330    } else {
11331        format!(
11332            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11333            state.instruction,
11334            inputs.join("\n")
11335        )
11336    }
11337}
11338
11339/// Does the winner have no commits ahead of the base it would land on?
11340/// Any failure to find out reads as "not empty": the merge then behaves as it
11341/// always did rather than refusing on a guess.
11342async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11343    let base = &state.base_branch;
11344    let mut against = base.clone();
11345    if mode == MergeMode::Pr {
11346        // A pull request lands on the remote's base, never the local branch
11347        // of the same name: if that cannot be read, "not empty" is the safe
11348        // answer.
11349        let remote = &state.config.merge.remote;
11350        let tracking = format!("{remote}/{base}");
11351        let fetched = git::fetch(repo, remote, base).await;
11352        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11353            against = tracking;
11354        } else {
11355            return false;
11356        }
11357    }
11358    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11359}
11360
11361/// Why nothing was opened for an empty winner, with what the task's own
11362/// references resolved to.
11363fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11364    let mut detail = format!(
11365        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11366         nothing to open a pull request for"
11367    );
11368    detail.push_str(
11369        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11370    );
11371    detail
11372}
11373
11374/// What the `Pr` merge does once it knows whether the branch already has an
11375/// open pull request.
11376#[derive(Debug, PartialEq, Eq)]
11377enum PrPlan {
11378    Create,
11379    Adopt { url: String, title: String },
11380    Stop(String),
11381}
11382
11383/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11384/// a failed lookup -> stop with the real reason. Never guesses.
11385fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11386    match found {
11387        Ok(land::OpenPr::None) => PrPlan::Create,
11388        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11389        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11390            "several open pull requests exist for this branch, not picking one: {}",
11391            urls.join(" ")
11392        )),
11393        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11394    }
11395}
11396
11397/// `gh pr create`, returning the PR url.
11398pub(crate) async fn gh_pr_create(
11399    cwd: &Path,
11400    base: &str,
11401    head: &str,
11402    title: &str,
11403    body: &str,
11404) -> Result<String> {
11405    let out = tokio::process::Command::new("gh")
11406        .args([
11407            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11408        ])
11409        .current_dir(cwd)
11410        .quiet()
11411        .stdin(std::process::Stdio::null())
11412        .output()
11413        .await
11414        .context("spawn gh")?;
11415    if out.status.success() {
11416        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11417    } else {
11418        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11419    }
11420}
11421
11422/// Never force deletion of uncommitted work, including unknown legacy trees.
11423async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11424    match std::fs::symlink_metadata(path) {
11425        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11426            // A previous cleanup may have removed the tree before a branch
11427            // deletion or state save failed. Unregister it if still present.
11428            git::worktree_remove_clean(repo, path).await?;
11429            return Ok(());
11430        }
11431        Err(e) => return Err(e.into()),
11432        Ok(_) => {}
11433    }
11434    if !git::is_clean(path).await? {
11435        bail!(
11436            "worktree has uncommitted work; refusing removal: {}",
11437            path.display()
11438        );
11439    }
11440    if !git::worktree_remove_clean(repo, path).await? {
11441        bail!("worktree removal refused: {}", path.display());
11442    }
11443    Ok(())
11444}
11445
11446/// Tear a run's worktrees and branches down.
11447///
11448/// `home` is where the updated `run.json` is saved (via
11449/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11450/// a housekeeping pass already has its own honest `home` handed to it, and
11451/// falling through to the global here would write back through whichever
11452/// directory some other process or test pinned into that `OnceLock` first,
11453/// not the one the caller actually resolved its `runs` and `state` from.
11454pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11455    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11456        bail!("refusing fold: {reason}");
11457    }
11458    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11459        bail!("candidate preservation incomplete; refusing cleanup");
11460    }
11461    // Serialize ordinary folding with the same starts/resumes as target-only
11462    // eviction. Otherwise a second janitor could remove the whole worktree
11463    // around a live output borrower even though target eviction was fenced.
11464    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11465    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11466    let _driver = crate::driver::maintenance(home, &state.id)?;
11467    let mut output_guards = Vec::new();
11468    for entry in std::fs::read_dir(state.worktree_root())
11469        .into_iter()
11470        .flatten()
11471        .flatten()
11472    {
11473        if entry.path().join(".git").is_file() {
11474            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11475        }
11476    }
11477    let repo = state.repo.clone();
11478    // CLI, Web and automatic cleanup all enter here, independently of
11479    // Runner::fold_losers. Protect deletion and shared config release too.
11480    let wait = repository_wait(state, None);
11481    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11482        Ok(lease) => lease,
11483        Err(e) => {
11484            state.save_under(home)?;
11485            return Err(e);
11486        }
11487    };
11488    // A merged PR proves only its published change. Preflight every source
11489    // before removing anything; a losing candidate or detached reviewer can
11490    // still retain independent commits or Git recovery checkpoints.
11491    let _source_outputs = if state.status == RunStatus::Merged
11492        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11493    {
11494        verify_merged_fold_sources(state, drop_winner, home).await?
11495    } else {
11496        Vec::new()
11497    };
11498    let root = state.worktree_root();
11499    let winner = state.tally.as_ref().map(|t| t.winner);
11500    let mut removed = Vec::new();
11501
11502    for i in 0..state.candidates.len() {
11503        let c = state.candidates[i].clone();
11504        let is_winner = Some(c.label) == winner;
11505        if is_winner && !drop_winner {
11506            continue;
11507        }
11508        let existed = c.worktree.exists();
11509        // A vanished directory can still have a Git registration that keeps
11510        // its branch checked out. Remove that registration before the branch.
11511        remove_preserved_worktree(&repo, &c.worktree).await?;
11512        if existed {
11513            removed.push(c.worktree.to_string_lossy().into_owned());
11514        }
11515        // A branch handed to a later run (and its pull request) is not this
11516        // run's to delete.
11517        let handed_over = state.released_branches.contains(&c.branch);
11518        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11519            if !git::branch_delete(&repo, &c.branch).await? {
11520                bail!("candidate branch removal refused");
11521            }
11522            removed.push(c.branch.clone());
11523        }
11524        state.candidates[i].folded = true;
11525    }
11526
11527    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11528        let path = name.path();
11529        let keep = !drop_winner
11530            && winner.is_some_and(|w| {
11531                path.file_name()
11532                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11533            });
11534        if keep {
11535            continue;
11536        }
11537        remove_preserved_worktree(&repo, &path).await?;
11538        removed.push(path.to_string_lossy().into_owned());
11539    }
11540
11541    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11542    // judge worktrees, so once the loop above has cleared all of them out,
11543    // the parent is a bare directory nobody else was ever going to remove -
11544    // git only ever managed what was inside it. Left alone, one of these
11545    // accumulates per fully-folded run; the operator's own machine had 74.
11546    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11547    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11548    // so does anything a run never claimed that happens to share the bay.
11549    remove_if_empty(&root);
11550
11551    if state.enabled_worktree_config && drop_winner {
11552        // A release, not a raw disable: some sibling run in this repository
11553        // may still hold its own reference (see `git::acquire_worktree_config`),
11554        // and only the last release actually turns the setting back off.
11555        git::release_worktree_config(&repo).await.ok();
11556        state.enabled_worktree_config = false;
11557    }
11558    state.save_under(home)?;
11559    Ok(removed)
11560}
11561
11562async fn verify_merged_fold_sources(
11563    state: &RunState,
11564    drop_winner: bool,
11565    home: &Path,
11566) -> Result<Vec<std::fs::File>> {
11567    let winner = state.tally.as_ref().map(|t| t.winner);
11568    let mut heads = BTreeSet::new();
11569    for candidate in &state.candidates {
11570        if Some(candidate.label) == winner && !drop_winner {
11571            continue;
11572        }
11573        match std::fs::symlink_metadata(&candidate.worktree) {
11574            Ok(_) => anyhow::ensure!(
11575                candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11576                "candidate recovery source is outside the inventoried run root: {}",
11577                candidate.worktree.display()
11578            ),
11579            Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11580            Err(error) => {
11581                return Err(error).context(format!(
11582                    "cannot inspect candidate recovery source: {}",
11583                    candidate.worktree.display()
11584                ));
11585            }
11586        }
11587        if git::branch_exists(&state.repo, &candidate.branch).await? {
11588            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11589            heads.extend(
11590                git::git(
11591                    &state.repo,
11592                    &["reflog", "show", "--format=%H", &candidate.branch],
11593                )
11594                .await?
11595                .lines()
11596                .map(str::to_owned),
11597            );
11598        }
11599    }
11600    let mut cargo_guards = Vec::new();
11601    let entries = match std::fs::read_dir(state.worktree_root()) {
11602        Ok(entries) => Some(entries),
11603        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11604        Err(error) => return Err(error.into()),
11605    };
11606    for entry in entries.into_iter().flatten() {
11607        let path = entry?.path();
11608        let metadata = std::fs::symlink_metadata(&path)?;
11609        anyhow::ensure!(
11610            metadata.is_dir() && !metadata.file_type().is_symlink(),
11611            "non-directory or symlink recovery source retained: {}",
11612            path.display()
11613        );
11614        if !drop_winner
11615            && winner.is_some_and(|w| {
11616                path.file_name()
11617                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11618            })
11619        {
11620            continue;
11621        }
11622        anyhow::ensure!(
11623            path.join(".git").is_file(),
11624            "unregistered recovery source retained: {}",
11625            path.display()
11626        );
11627        anyhow::ensure!(
11628            git::git(
11629                &path,
11630                &[
11631                    "status",
11632                    "--porcelain",
11633                    "--untracked-files=all",
11634                    "--ignored",
11635                    "--",
11636                    ".",
11637                    ":(exclude)target"
11638                ]
11639            )
11640            .await?
11641            .is_empty(),
11642            "source changes or ignored recovery files retained: {}",
11643            path.display()
11644        );
11645        match std::fs::symlink_metadata(path.join("target")) {
11646            Ok(_) => {
11647                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11648            }
11649            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11650            Err(error) => return Err(error.into()),
11651        }
11652        for checkpoint in [
11653            "rebase-merge",
11654            "rebase-apply",
11655            "sequencer",
11656            "CHERRY_PICK_HEAD",
11657            "MERGE_HEAD",
11658            "REBASE_HEAD",
11659            "REVERT_HEAD",
11660            "BISECT_LOG",
11661            "AUTO_MERGE",
11662            "MERGE_MSG",
11663        ] {
11664            let location = git::git(
11665                &path,
11666                &[
11667                    "rev-parse",
11668                    "--path-format=absolute",
11669                    "--git-path",
11670                    checkpoint,
11671                ],
11672            )
11673            .await?;
11674            anyhow::ensure!(
11675                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11676                if e.kind() == std::io::ErrorKind::NotFound),
11677                "Git recovery checkpoint retained: {checkpoint}"
11678            );
11679        }
11680        heads.insert(git::rev_parse(&path, "HEAD").await?);
11681        heads.extend(
11682            git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11683                .await?
11684                .lines()
11685                .map(str::to_owned),
11686        );
11687        if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11688            heads.insert(original);
11689        }
11690    }
11691    if !heads.is_empty() {
11692        // Never trust a run-wide merge label or the local checkout's branch.
11693        // Pin the fetched base once. A squash does not preserve commit IDs:
11694        // prove the winner's cumulative patch before accepting its ancestors.
11695        let reference = format!(
11696            "refs/remotes/{}/{}",
11697            state.config.merge.remote, state.base_branch
11698        );
11699        let base = git::rev_parse(&state.repo, &reference)
11700            .await
11701            .context("cannot prove merged sources against the fetched base")?;
11702        let mut squash_winner = None;
11703        let mut checked_squash = false;
11704        for head in heads {
11705            if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11706                .await?
11707                .ok()
11708            {
11709                continue;
11710            }
11711            if !checked_squash {
11712                squash_winner = proven_squash_winner(state, &base).await?;
11713                checked_squash = true;
11714            }
11715            let incorporated = if let Some(winner) = &squash_winner {
11716                git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11717                    .await?
11718                    .ok()
11719            } else {
11720                false
11721            };
11722            anyhow::ensure!(
11723                incorporated,
11724                "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11725            );
11726        }
11727    }
11728    Ok(cargo_guards)
11729}
11730
11731/// Called only for a confirmed merged run. Its label admits proof, but does
11732/// not prove that any particular local branch or recovery checkpoint landed.
11733/// Match the complete binary patch, including modes and blob IDs, against a
11734/// single-parent commit in the pinned fetched history. This works after later
11735/// base commits too, without trusting the current base tree to still match.
11736async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11737    let Some(winner) = state.winner() else {
11738        return Ok(None);
11739    };
11740    let head = git::rev_parse(&state.repo, &winner.branch).await?;
11741    if !git::git_raw(
11742        &state.repo,
11743        &["merge-base", "--is-ancestor", &state.base_commit, &head],
11744    )
11745    .await?
11746    .ok()
11747    {
11748        return Ok(None);
11749    }
11750    let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11751    if patch.is_empty() {
11752        return Ok(None);
11753    }
11754    let range = format!("{}..{base}", state.base_commit);
11755    let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11756    for commit in commits.lines() {
11757        let parent = format!("{commit}^");
11758        if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11759            return Ok(Some(head));
11760        }
11761    }
11762    Ok(None)
11763}
11764
11765async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11766    git::git(
11767        repo,
11768        &[
11769            "-c",
11770            "core.quotePath=true",
11771            "diff",
11772            "--binary",
11773            "--full-index",
11774            "--no-renames",
11775            "--no-ext-diff",
11776            "--no-textconv",
11777            "--no-color",
11778            "--ignore-submodules=none",
11779            "--submodule=short",
11780            from,
11781            to,
11782            "--",
11783        ],
11784    )
11785    .await
11786}
11787
11788/// Remove `dir` if it exists and has nothing in it.
11789///
11790/// Best-effort and silent by design: a directory that is not empty (a run
11791/// whose winner is still parked there, a stray file some other process left)
11792/// is exactly the case this must refuse, and a directory that is already gone
11793/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11794/// already refuses a non-empty directory, so the emptiness check below is
11795/// belt, not suspenders - it is what keeps this from ever attempting the
11796/// removal in the case that matters, rather than trusting `remove_dir`'s
11797/// error path to have no side effects if it ever changed.
11798fn remove_if_empty(dir: &Path) {
11799    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11800        std::fs::remove_dir(dir).ok();
11801    }
11802}
11803
11804/// Severity of the worst open finding in the last review round, for reporting.
11805pub fn worst_open(state: &RunState) -> Option<Severity> {
11806    state
11807        .reviews
11808        .last()?
11809        .reviews
11810        .iter()
11811        .flat_map(|r| r.findings.iter())
11812        .map(|f| f.severity)
11813        .max()
11814}
11815
11816/// `[worktree] setup` hides its products per worktree, which needs
11817/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11818/// worktrees, like the hook's; every place that runs setup calls this first.
11819async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11820    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11821        git::acquire_worktree_config(repo).await?;
11822        state.enabled_worktree_config = true;
11823    }
11824    Ok(())
11825}
11826
11827#[cfg(test)]
11828mod tests {
11829    #[tokio::test]
11830    async fn launcher_base_fetch_requires_remote_success() {
11831        let tmp = tempfile::tempdir().unwrap();
11832        let repo = tmp.path().join("repo");
11833        let remote = tmp.path().join("remote");
11834        std::fs::create_dir(&repo).unwrap();
11835        std::fs::create_dir(&remote).unwrap();
11836        git::git(&remote, &["init", "--bare", "-b", "main"])
11837            .await
11838            .unwrap();
11839        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11840        git::git(
11841            &repo,
11842            &[
11843                "-c",
11844                "user.name=Fixture",
11845                "-c",
11846                "user.email=fixture@example.invalid",
11847                "commit",
11848                "--allow-empty",
11849                "-m",
11850                "fixture",
11851            ],
11852        )
11853        .await
11854        .unwrap();
11855        git::git(
11856            &repo,
11857            &["remote", "add", "origin", remote.to_str().unwrap()],
11858        )
11859        .await
11860        .unwrap();
11861        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11862        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11863        assert_eq!(
11864            resolve_base(&repo, "main", "origin", None)
11865                .await
11866                .unwrap()
11867                .commit,
11868            head
11869        );
11870        #[cfg(unix)]
11871        {
11872            let policy: crate::config::Coordination = toml::from_str(
11873                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11874            ).unwrap();
11875            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11876            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11877            let held = crate::repo_lock::SeatLease::operation_with(
11878                &repo,
11879                Some(&policy),
11880                &crate::repo_lock::WaitPolicy::default(),
11881            )
11882            .await
11883            .unwrap();
11884            assert!(held.is_some());
11885            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11886            tokio::pin!(fetch);
11887            assert!(
11888                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11889                    .await
11890                    .is_err()
11891            );
11892            assert!(
11893                !repo.join(".git/FETCH_HEAD").exists(),
11894                "no fetch before admission"
11895            );
11896            drop(held);
11897            let fetched = fetch.await.unwrap();
11898            assert_eq!(fetched.commit, head);
11899            assert_eq!(fetched.remote, "origin");
11900            assert_eq!(fetched.branch, "main");
11901        }
11902        // A surviving tracking ref is not successful fetch evidence.
11903        git::git(
11904            &repo,
11905            &[
11906                "remote",
11907                "set-url",
11908                "origin",
11909                tmp.path().join("missing").to_str().unwrap(),
11910            ],
11911        )
11912        .await
11913        .unwrap();
11914        let error = resolve_base(&repo, "main", "origin", None)
11915            .await
11916            .unwrap_err()
11917            .to_string();
11918        assert!(error.contains("cannot read origin/main"));
11919        assert!(error.contains("refusing to branch off"));
11920    }
11921
11922    #[test]
11923    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11924        let leaked = vec!["chore(deps): update a crate".to_owned()];
11925        let own = "fix(daemon): apply a chosen action";
11926        assert!(should_retitle("", own, &leaked));
11927        assert!(should_retitle(
11928            &format!("{REVIEW_PROMPT_OPENING} `x`"),
11929            own,
11930            &leaked
11931        ));
11932        assert!(should_retitle(
11933            "chore: land candidate A of run 1",
11934            own,
11935            &leaked
11936        ));
11937        assert!(should_retitle(
11938            "magi: candidate A (uncommitted work)",
11939            own,
11940            &leaked
11941        ));
11942        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11943        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11944        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11945    }
11946
11947    #[test]
11948    fn pr_merge_plan_creates_adopts_or_stops() {
11949        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11950        assert_eq!(
11951            pr_merge_plan(Ok(land::OpenPr::One {
11952                url: "u".into(),
11953                title: "t".into()
11954            })),
11955            PrPlan::Adopt {
11956                url: "u".into(),
11957                title: "t".into()
11958            }
11959        );
11960        let PrPlan::Stop(many) =
11961            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11962        else {
11963            panic!("many must stop");
11964        };
11965        assert!(many.contains('a') && many.contains('b'));
11966        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11967            panic!("a failed lookup must stop");
11968        };
11969        assert!(err.contains("bad token"));
11970    }
11971
11972    use super::*;
11973    use crate::run::GateStatus;
11974    use std::collections::BTreeMap;
11975    use std::time::Duration;
11976
11977    fn conductor() -> AgentSpec {
11978        AgentSpec {
11979            id: "conductor".to_owned(),
11980            kind: crate::config::AgentKind::Command,
11981            model: None,
11982            command: vec!["true".to_owned()],
11983            extra_args: Vec::new(),
11984            env: BTreeMap::new(),
11985            prompt_delivery: None,
11986        }
11987    }
11988
11989    fn spec(id: &str) -> AgentSpec {
11990        AgentSpec {
11991            id: id.to_owned(),
11992            kind: crate::config::AgentKind::Command,
11993            model: None,
11994            command: vec!["true".to_owned()],
11995            extra_args: Vec::new(),
11996            env: BTreeMap::new(),
11997            prompt_delivery: None,
11998        }
11999    }
12000
12001    fn ids(xs: &[&str]) -> BTreeSet<String> {
12002        xs.iter().map(|s| (*s).to_owned()).collect()
12003    }
12004
12005    #[test]
12006    fn pick_successor_skips_an_agent_another_seat_holds() {
12007        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12008        let roster = [spec("a"), spec("b"), spec("c")];
12009        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12010        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12011    }
12012
12013    #[test]
12014    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12015        // The other seat started on c but was handed to d; c is free again.
12016        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12017        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12018        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12019        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12020        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12021    }
12022
12023    #[test]
12024    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12025        let roster = [spec("a"), spec("b")];
12026        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12027        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12028        let carried = ids(&["b"]);
12029        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12030        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12031    }
12032
12033    #[test]
12034    fn pick_successor_returns_none_without_any_untried_successor() {
12035        let roster = [spec("a"), spec("b")];
12036        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12037        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12038        let carried = ids(&["a"]);
12039        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12040    }
12041
12042    #[test]
12043    fn pick_successor_rescue_avoids_another_seats_occupant() {
12044        // b is a carried failure and free; a is held by the other seat.
12045        let roster = [spec("a"), spec("b"), spec("c")];
12046        let carried = ids(&["b", "c"]);
12047        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12048        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12049    }
12050
12051    #[test]
12052    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12053        let roster = [spec("a"), spec("b"), spec("c")];
12054        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12055        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12056    }
12057
12058    #[test]
12059    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12060        let roster = [spec("a"), spec("b"), spec("c")];
12061        let failed = ids(&["a", "b", "c"]);
12062        // Rescue looks at the whole roster, once per id, then runs out.
12063        let mut tried = ids(&["b"]);
12064        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12065        assert_eq!(first.id, "a");
12066        tried.insert(first.id.clone());
12067        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12068        assert_eq!(second.id, "c");
12069        tried.insert(second.id.clone());
12070        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12071    }
12072
12073    #[test]
12074    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12075        let roster = [spec("a"), spec("b")];
12076        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12077        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12078    }
12079
12080    #[test]
12081    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12082        let roster = [spec("a"), spec("b"), spec("c")];
12083        let h = SeatHistory {
12084            failed: ids(&["a"]),
12085            last_ok: Some("b".to_owned()),
12086            last_fail: None,
12087        };
12088        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12089        // A last answerer that left the roster, or later failed, is ignored.
12090        let gone = SeatHistory {
12091            last_ok: Some("zzz".to_owned()),
12092            ..h.clone()
12093        };
12094        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12095        let failed = SeatHistory {
12096            failed: ids(&["a", "b"]),
12097            last_ok: Some("b".to_owned()),
12098            last_fail: None,
12099        };
12100        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12101    }
12102
12103    #[test]
12104    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12105        let first = SeatState::new("review-1", "alpha", 7);
12106        let next = handover_seat("review-1", "gamma", 7);
12107        assert_ne!(first.claude_session, next.claude_session);
12108    }
12109
12110    #[test]
12111    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12112        let mut state = state_with_summary("x", "y");
12113        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12114        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12115        assert_ne!(a.claude_session, b.claude_session);
12116    }
12117
12118    #[test]
12119    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12120        let roster = [spec("a"), spec("b")];
12121        let h = SeatHistory {
12122            failed: ids(&["a", "b"]),
12123            ..SeatHistory::default()
12124        };
12125        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12126        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12127        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12128    }
12129
12130    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12131    // fallback loop depends on to terminate: it must walk forward from the
12132    // seat's own position, never restart at the front of the roster, and it
12133    // must never hand back an id already tried, however many times that id
12134    // happens to appear.
12135
12136    #[test]
12137    fn failure_signature_ignores_numbers_and_paths() {
12138        assert_eq!(
12139            failure_signature("exited with Some(2) and no usable output"),
12140            failure_signature("exited with Some(137) and no usable output")
12141        );
12142        assert_eq!(
12143            failure_signature("process exited with no usable output (exit code 2)"),
12144            failure_signature("process exited with no usable output (exit code 137)")
12145        );
12146        for (old, new) in [
12147            (
12148                "exited with Some(2) and no usable output",
12149                "process exited with no usable output (exit code 2)",
12150            ),
12151            (
12152                "exited with None and no usable output",
12153                "process exited with no usable output (exit code unavailable)",
12154            ),
12155        ] {
12156            let previous = FailClass::Other(failure_signature(old));
12157            let current = FailClass::Other(failure_signature(new));
12158            assert_eq!(previous, current, "resumption retains the failure class");
12159            assert!(!should_hand_over(Some(&previous), &current));
12160        }
12161        assert_eq!(
12162            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12163            failure_signature("cannot open /var/x.txt: denied")
12164        );
12165        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12166    }
12167
12168    #[test]
12169    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12170        let other = FailClass::Other("x".into());
12171        assert!(should_hand_over(None, &FailClass::Quota));
12172        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12173        assert!(should_hand_over(
12174            Some(&FailClass::Timeout),
12175            &FailClass::Timeout
12176        ));
12177        assert!(should_hand_over(None, &other));
12178        assert!(!should_hand_over(Some(&other), &other));
12179        assert!(should_hand_over(
12180            Some(&other),
12181            &FailClass::Other("y".into())
12182        ));
12183        // A quota or timeout in between ends the run of identical failures.
12184        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12185        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12186    }
12187
12188    #[test]
12189    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12190        let a = SeatState::new("judge-1", "alpha", 7);
12191        let b = handover_seat("judge-1", "beta", 7);
12192        assert_ne!(a.claude_session, b.claude_session);
12193        assert_eq!(b.turns, 0);
12194    }
12195
12196    #[test]
12197    fn a_timeout_is_classified_apart_from_other_failures() {
12198        assert_eq!(
12199            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12200            Some(FailClass::Timeout)
12201        );
12202        assert!(matches!(
12203            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12204            Some(FailClass::Other(_))
12205        ));
12206    }
12207
12208    #[test]
12209    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12210        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12211        let tried = BTreeSet::from(["beta".to_owned()]);
12212        // beta sits at index 1; the next candidate is gamma, never alpha —
12213        // which is very likely a different candidate slot's own agent.
12214        let next = next_untried_in_roster(&roster, 1, &tried);
12215        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12216    }
12217
12218    #[test]
12219    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12220        let roster = vec![spec("alpha"), spec("beta")];
12221        let tried = BTreeSet::from(["beta".to_owned()]);
12222        // beta is the roster's last entry: nothing follows it, and alpha —
12223        // earlier in the roster, almost certainly a different candidate
12224        // slot's own agent — must not be reached by wrapping back to it.
12225        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12226    }
12227
12228    #[test]
12229    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12230        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12231        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12232        // beta (index 1) and gamma (index 2, the only entry after it) have
12233        // both been tried; alpha (index 0) never has, but it comes before
12234        // beta's own position, so there is nothing further for this seat.
12235        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12236    }
12237
12238    #[test]
12239    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12240        let roster = vec![spec("a"), spec("a"), spec("b")];
12241        let tried = BTreeSet::from(["a".to_owned()]);
12242        let next = next_untried_in_roster(&roster, 0, &tried);
12243        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12244    }
12245
12246    #[test]
12247    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12248        let roster = vec![spec("a"), spec("b")];
12249        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12250        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12251    }
12252
12253    #[test]
12254    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12255        let dir = tempfile::tempdir().unwrap();
12256        let bay = dir.path().join("ffff");
12257
12258        // Not there yet: nothing to do, nothing to panic on.
12259        remove_if_empty(&bay);
12260        assert!(!bay.exists());
12261
12262        // Something still inside - the winner's worktree, or a stray file -
12263        // keeps the directory standing.
12264        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12265        remove_if_empty(&bay);
12266        assert!(bay.exists(), "non-empty directory must survive");
12267
12268        // Once the last entry is gone, so is the directory itself.
12269        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12270        remove_if_empty(&bay);
12271        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12272    }
12273
12274    // `round_is_clean` is the exact decision this task fixed: a round with a
12275    // seat that never answered must not read the same as a round every seat
12276    // actually reviewed. These are deterministic and process-free by design —
12277    // the equivalent end-to-end check (a real reviewer timing out under a
12278    // live graph run) is a genuine race against wall-clock contention, and a
12279    // spawn slow enough to blow even a generous budget under a loaded test
12280    // run must not turn this specific regression check flaky.
12281
12282    #[test]
12283    fn a_full_panel_that_found_nothing_is_clean() {
12284        assert!(round_is_clean(
12285            0,
12286            true,
12287            2,
12288            2,
12289            0,
12290            IncompleteReviewPolicy::Block
12291        ));
12292    }
12293
12294    #[test]
12295    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12296        assert!(!round_is_clean(
12297            0,
12298            true,
12299            1,
12300            2,
12301            0,
12302            IncompleteReviewPolicy::Block
12303        ));
12304    }
12305
12306    #[test]
12307    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12308        assert!(!round_is_clean(
12309            1,
12310            true,
12311            1,
12312            2,
12313            0,
12314            IncompleteReviewPolicy::Warn
12315        ));
12316    }
12317
12318    #[test]
12319    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12320        assert!(round_is_clean(
12321            0,
12322            true,
12323            1,
12324            2,
12325            0,
12326            IncompleteReviewPolicy::Warn
12327        ));
12328    }
12329
12330    #[test]
12331    fn a_full_panel_with_an_open_finding_is_not_clean() {
12332        assert!(!round_is_clean(
12333            1,
12334            true,
12335            2,
12336            2,
12337            0,
12338            IncompleteReviewPolicy::Block
12339        ));
12340    }
12341
12342    #[test]
12343    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12344        assert!(!round_is_clean(
12345            0,
12346            false,
12347            2,
12348            2,
12349            0,
12350            IncompleteReviewPolicy::Block
12351        ));
12352    }
12353
12354    // The stall this task closes: under the default `block` policy, a seat
12355    // missing only because it was rate limited must not force a wait for a
12356    // session limit that will not lift by the next round. `round_is_clean`
12357    // is where that quorum carve-out lives; the review loop around it never
12358    // changes what a reviewer's vote or a finding's severity means.
12359
12360    #[test]
12361    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12362        // 1 of 2 answered, and the one missing was quota'd — the exact
12363        // "review-2 rate limited (quota)" shape from the field report.
12364        assert!(round_is_clean(
12365            0,
12366            true,
12367            1,
12368            2,
12369            1,
12370            IncompleteReviewPolicy::Block
12371        ));
12372    }
12373
12374    #[test]
12375    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12376        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12377        // not a quota loss (`quota_missing` stays 0) — worth another try.
12378        assert!(!round_is_clean(
12379            0,
12380            true,
12381            1,
12382            2,
12383            0,
12384            IncompleteReviewPolicy::Block
12385        ));
12386    }
12387
12388    #[test]
12389    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12390        assert!(!round_is_clean(
12391            1,
12392            true,
12393            1,
12394            2,
12395            1,
12396            IncompleteReviewPolicy::Block
12397        ));
12398        assert!(!round_is_clean(
12399            0,
12400            false,
12401            1,
12402            2,
12403            1,
12404            IncompleteReviewPolicy::Block
12405        ));
12406    }
12407
12408    #[test]
12409    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12410        // Every seat quota'd, nobody answered: there is no panel to decide
12411        // on, so this must fall through to the existing block-and-retry
12412        // fallback rather than call an unreviewed patch clean.
12413        assert!(!round_is_clean(
12414            0,
12415            true,
12416            0,
12417            2,
12418            2,
12419            IncompleteReviewPolicy::Block
12420        ));
12421    }
12422
12423    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12424        CommandOutcome {
12425            command: "test".to_owned(),
12426            code,
12427            output_tail: String::new(),
12428            duration_ms: 0,
12429            resource_blocked,
12430        }
12431    }
12432
12433    #[test]
12434    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12435        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12436        assert!(
12437            !verify_inconclusive(&[outcome(Some(1), false)]),
12438            "an ordinary failure is still evidence about the patch"
12439        );
12440        assert!(verify_inconclusive(&[outcome(None, true)]));
12441        assert!(
12442            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12443            "one inconclusive outcome taints the whole batch"
12444        );
12445        assert!(!verify_inconclusive(&[]));
12446    }
12447
12448    #[tokio::test]
12449    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12450        // Alive for the first two checks, then dead - confirms the loop
12451        // actually re-polls rather than deciding once and sleeping out the
12452        // ceiling regardless.
12453        let calls = std::sync::atomic::AtomicUsize::new(0);
12454        let started = Instant::now();
12455        wait_for_pids_with(
12456            &[123],
12457            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12458            Duration::from_millis(5),
12459            Duration::from_secs(5),
12460        )
12461        .await;
12462        assert!(
12463            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12464            "must keep checking rather than deciding on the first answer"
12465        );
12466        assert!(
12467            started.elapsed() < Duration::from_secs(1),
12468            "must return the moment it is confirmed dead, not wait out the ceiling"
12469        );
12470    }
12471
12472    #[tokio::test]
12473    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12474        let started = Instant::now();
12475        wait_for_pids_with(
12476            &[123],
12477            |_| true, // never reports dead
12478            Duration::from_millis(5),
12479            Duration::from_millis(30),
12480        )
12481        .await;
12482        let elapsed = started.elapsed();
12483        assert!(
12484            elapsed >= Duration::from_millis(30),
12485            "must not give up before its own ceiling: {elapsed:?}"
12486        );
12487        assert!(
12488            elapsed < Duration::from_secs(1),
12489            "must not wait past its own ceiling either: {elapsed:?}"
12490        );
12491    }
12492
12493    #[tokio::test]
12494    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12495        let started = Instant::now();
12496        wait_for_pids_with(
12497            &[],
12498            |_| true,
12499            Duration::from_secs(5),
12500            Duration::from_secs(5),
12501        )
12502        .await;
12503        assert!(
12504            started.elapsed() < Duration::from_millis(200),
12505            "an empty pid list has nothing to confirm"
12506        );
12507    }
12508
12509    // `review_conclusion` is the exact decision the review hand-off task
12510    // fixed: a round budget spent (or a tree that stopped moving) must not
12511    // collapse into `Blocked` regardless of what verification actually
12512    // said. Deterministic and process-free for the same reason the
12513    // `round_is_clean` family above is.
12514    fn review_round(
12515        clean: bool,
12516        blocking: usize,
12517        answered: usize,
12518        expected: usize,
12519        progressed: bool,
12520        e2e_ok: bool,
12521    ) -> ReviewRound {
12522        ReviewRound {
12523            round: 1,
12524            head: "h".to_owned(),
12525            verified_head: None,
12526            verified_at: None,
12527            reviews: Vec::new(),
12528            e2e: vec![CommandOutcome {
12529                command: "test".to_owned(),
12530                code: Some(if e2e_ok { 0 } else { 1 }),
12531                output_tail: String::new(),
12532                duration_ms: 0,
12533                resource_blocked: false,
12534            }],
12535            verify_retried: false,
12536            e2e_deferred: false,
12537            e2e_defer_reason: None,
12538            fix: None,
12539            blocking,
12540            answered,
12541            expected,
12542            clean,
12543            progressed,
12544            vote_split: false,
12545            reconsideration: Vec::new(),
12546            verdict: None,
12547        }
12548    }
12549
12550    #[test]
12551    fn review_conclusion_is_none_when_nothing_has_run() {
12552        assert_eq!(review_conclusion(&[], 3), None);
12553    }
12554
12555    #[test]
12556    fn review_conclusion_is_none_while_rounds_remain() {
12557        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12558        assert_eq!(review_conclusion(&rounds, 3), None);
12559    }
12560
12561    #[test]
12562    fn review_conclusion_is_gating_once_a_round_is_clean() {
12563        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12564        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12565    }
12566
12567    #[test]
12568    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12569        let rounds = vec![
12570            review_round(false, 1, 2, 2, true, true),
12571            review_round(false, 1, 2, 2, true, true),
12572        ];
12573        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12574    }
12575
12576    #[test]
12577    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12578        let rounds = vec![
12579            review_round(false, 1, 2, 2, true, true),
12580            review_round(false, 1, 2, 2, true, false),
12581        ];
12582        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12583    }
12584
12585    #[test]
12586    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12587        // Magi never got a command to run against this round's own head — a
12588        // resource-blocked attempt, not a red one — so this must never
12589        // settle on `Blocked` the way a genuine e2e failure would. `None`
12590        // here is what tells `Runner::review_loop` to retry the check
12591        // itself rather than trust this cheap recomputation with a verdict
12592        // it cannot actually produce.
12593        let mut blocked = review_round(false, 1, 2, 2, true, false);
12594        blocked.e2e[0].resource_blocked = true;
12595        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12596        assert_eq!(review_conclusion(&rounds, 2), None);
12597    }
12598
12599    #[test]
12600    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12601        // Missing input, not a verified tree — never a hand-off candidate.
12602        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12603        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12604    }
12605
12606    #[test]
12607    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12608        let rounds = vec![
12609            review_round(false, 1, 2, 2, false, true),
12610            review_round(false, 1, 2, 2, false, true),
12611        ];
12612        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12613    }
12614
12615    fn secs(n: u64) -> Duration {
12616        Duration::from_secs(n)
12617    }
12618
12619    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12620    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12621    #[tokio::test]
12622    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12623        ask_test_home();
12624        let tmp = tempfile::tempdir().unwrap();
12625        let repo = tmp.path().join("repo");
12626        std::fs::create_dir_all(&repo).unwrap();
12627        init_repo(&repo);
12628        let g = |args: &[&str]| {
12629            let out = std::process::Command::new("git")
12630                .args(args)
12631                .current_dir(&repo)
12632                .quiet()
12633                .output()
12634                .unwrap();
12635            assert!(
12636                out.status.success(),
12637                "{}",
12638                String::from_utf8_lossy(&out.stderr)
12639            );
12640        };
12641        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12642        g(&["checkout", "-b", "historical"]);
12643        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12644        g(&["add", "."]);
12645        g(&["commit", "-m", "old alternative"]);
12646        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12647        g(&["checkout", "-b", "magi/task-selected", &base]);
12648        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12649        g(&["add", "."]);
12650        g(&["commit", "-m", "measurement infrastructure"]);
12651        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12652        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12653        g(&["add", "."]);
12654        g(&["commit", "-m", "dependent implementation"]);
12655        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12656        g(&["checkout", "main"]);
12657        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12658        g(&["add", "."]);
12659        g(&["commit", "-m", "unrelated base change"]);
12660        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12661        let origin = tmp.path().join("origin.git");
12662        g(&[
12663            "clone",
12664            "--bare",
12665            repo.to_str().unwrap(),
12666            origin.to_str().unwrap(),
12667        ]);
12668        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12669        g(&["fetch", "origin"]);
12670        let selection = crate::pr_recovery::Selection {
12671            url: "https://example.test/project/pull/1".into(),
12672            branch: "magi/task-selected".into(),
12673            head: head.clone(),
12674            source_head: None,
12675            source_run: None,
12676            source_proof: None,
12677            disposition: Some(crate::pr_recovery::Disposition::Review),
12678            instruction: "preserve measured behavior".into(),
12679            approved_at: jiff::Timestamp::now(),
12680        };
12681        let mut template = runner_at(RunStatus::Reviewing);
12682        template.state.repo = repo.clone();
12683        template.state.base_commit = current.clone();
12684        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12685        template.state.origin = Some(crate::run::Origin::queue("selected"));
12686        template.state.pr_head = Some(selection.branch.clone());
12687        template.state.pr_recovery = Some(selection.clone());
12688        let mut runner =
12689            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12690                .await
12691                .unwrap();
12692        let wt = runner.state.winner().unwrap().worktree.clone();
12693        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12694        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12695        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12696        assert!(runner.state.seeds.is_empty());
12697        assert_eq!(runner.state.instruction, "preserve measured behavior");
12698        runner.sync_to_base().await.unwrap();
12699        assert_eq!(
12700            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12701            "prerequisite"
12702        );
12703        assert_eq!(
12704            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12705            "measured implementation"
12706        );
12707        assert_eq!(
12708            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12709            "current main change"
12710        );
12711        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12712        assert_eq!(
12713            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12714            head
12715        );
12716
12717        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12718        let mut partial = runner_at(RunStatus::Reviewing);
12719        partial.state.repo = repo.clone();
12720        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12721        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12722            run: Some("exit 1".into()),
12723            ..Default::default()
12724        }];
12725        let selection = runner.state.pr_recovery.clone().unwrap();
12726        partial.state.pr_recovery = Some(selection.clone());
12727        partial.state.pr_head = Some(selection.branch.clone());
12728        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12729        let id = partial.state.id.clone();
12730        let error = match Runner::open_selected(
12731            &repo,
12732            partial.state,
12733            partial.roles,
12734            1,
12735            runner.state.base_commit.clone(),
12736            selection,
12737        )
12738        .await
12739        {
12740            Ok(_) => panic!("setup must fail"),
12741            Err(e) => e,
12742        };
12743        assert_eq!(
12744            error
12745                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12746                .unwrap()
12747                .run,
12748            id
12749        );
12750        let failed = RunState::load(&id).unwrap();
12751        assert_eq!(failed.status, RunStatus::Blocked);
12752        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12753        assert!(Runner::resume(&id).is_err());
12754        runner.state.status = RunStatus::Ready;
12755        runner.state.driver_exited = true;
12756        runner.state.save().unwrap();
12757        let mut retry = runner_at(RunStatus::Reviewing);
12758        retry.state.repo = repo.clone();
12759        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12760        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12761        reselected.approved_at = jiff::Timestamp::now();
12762        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12763        retry.state.pr_head = Some(reselected.branch.clone());
12764        retry.state.pr_recovery = Some(reselected.clone());
12765        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12766            .await
12767            .unwrap();
12768        let restarted = Runner::open_selected(
12769            &repo,
12770            retry.state,
12771            retry.roles,
12772            1,
12773            runner.state.base_commit.clone(),
12774            reselected,
12775        )
12776        .await
12777        .unwrap();
12778        assert_eq!(
12779            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12780                .await
12781                .unwrap(),
12782            head
12783        );
12784        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12785        assert_eq!(
12786            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12787            head
12788        );
12789    }
12790
12791    #[tokio::test]
12792    #[cfg(unix)]
12793    async fn fallback_and_interrupted_restoration_require_coordination() {
12794        use std::os::fd::AsRawFd as _;
12795        ask_test_home();
12796        let tmp = tempfile::tempdir().unwrap();
12797        let mut runner = registered_runner(tmp.path()).await;
12798        runner.state.config.graph.timeout_resource_wait = 1;
12799        let winner = runner.state.candidates[0].clone();
12800        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12801        let leftover = winner.worktree.join("leftover");
12802        std::fs::write(&leftover, "untrusted edits").unwrap();
12803        let next = tmp.path().join("next-fixer");
12804        let mut failed = spec("first");
12805        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12806        let mut successor = spec("second");
12807        successor.command = vec![
12808            "sh".into(),
12809            "-c".into(),
12810            format!("touch '{}'; echo done", next.display()),
12811        ];
12812        let attempts = vec![
12813            (failed.clone(), "first".into()),
12814            (successor.clone(), "second".into()),
12815        ];
12816        let lock_path = runner.state.repo.join(".git/updater.lock");
12817        let external = std::fs::File::create(&lock_path).unwrap();
12818        assert_eq!(
12819            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12820            0
12821        );
12822        let artifacts = tmp.path().join("artifacts");
12823        let build = |spec, seat| SeatJob {
12824            spec,
12825            seat,
12826            cwd: winner.worktree.clone(),
12827            prompt: "fixture".into(),
12828            timeout: Duration::from_secs(10),
12829            allow_write: true,
12830            sessions: false,
12831            artifacts: artifacts.clone(),
12832            stem: "fix".into(),
12833            handover: None,
12834        };
12835        let (_, _, out) = runner
12836            .ask_fixer_among(
12837                attempts.clone(),
12838                "graft",
12839                None,
12840                Some((&winner.worktree, &head)),
12841                build,
12842            )
12843            .await;
12844        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12845        assert!(leftover.exists(), "contention must prevent clean");
12846        assert!(!next.exists());
12847        assert!(runner.state.handovers.is_empty());
12848        // Early interrupted-repair rollback uses the same protected route.
12849        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12850            from_run: "earlier".into(),
12851            generation: 1,
12852            mode: crate::recovery::Mode::VerifyRepair,
12853            reasons: Vec::new(),
12854            reviewed_head: None,
12855            rounds: Vec::new(),
12856            open_findings: Vec::new(),
12857            evidence: None,
12858            criteria_inherited: true,
12859            repair: crate::recovery::Repair::Started {
12860                before: head.clone(),
12861            },
12862        });
12863        assert!(
12864            runner
12865                .recovery_repair(&winner)
12866                .await
12867                .unwrap_err()
12868                .is::<crate::repo_lock::ResourceBlocked>()
12869        );
12870        assert!(matches!(
12871            runner.state.recovery.as_ref().unwrap().repair,
12872            crate::recovery::Repair::Started { .. }
12873        ));
12874        assert!(leftover.exists());
12875        drop(external);
12876        runner.recovery_repair(&winner).await.unwrap();
12877        assert!(!leftover.exists());
12878        assert!(matches!(
12879            runner.state.recovery.as_ref().unwrap().repair,
12880            crate::recovery::Repair::Done { .. }
12881        ));
12882        // A hard lock-open failure also leaves HEAD and untracked work intact.
12883        std::fs::remove_file(&lock_path).unwrap();
12884        std::fs::create_dir(&lock_path).unwrap();
12885        std::fs::write(&leftover, "untrusted edits").unwrap();
12886        let (_, _, out) = runner
12887            .ask_fixer_among(
12888                attempts,
12889                "graft",
12890                None,
12891                Some((&winner.worktree, &head)),
12892                build,
12893            )
12894            .await;
12895        assert!(matches!(out, AgentOutcome::Failed(_)));
12896        assert!(leftover.exists());
12897        assert!(!next.exists());
12898        assert_eq!(
12899            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12900            head
12901        );
12902    }
12903
12904    #[cfg(unix)]
12905    async fn registered_runner(root: &Path) -> Runner {
12906        ask_test_home();
12907        let repo = root.join("repo");
12908        std::fs::create_dir_all(&repo).unwrap();
12909        init_repo(&repo);
12910        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12911        let mut runner = runner_at(RunStatus::Prep);
12912        runner.state.repo = repo.clone();
12913        runner.state.config.coordination = crate::config::test_coordination();
12914        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12915        runner.state.config.graph.worktree_root = Some(root.join("wt"));
12916        runner.state.config.graph.advise = false;
12917        runner.roles.implementers = vec![spec("implement")];
12918        runner.roles.judges = vec![spec("judge")];
12919        runner.prep().await.unwrap();
12920        runner
12921    }
12922
12923    #[tokio::test]
12924    #[cfg(unix)]
12925    async fn parked_implementer_persists_completed_work() {
12926        for registered in [false, true] {
12927            let tmp = tempfile::tempdir().unwrap();
12928            let mut runner = registered_runner(tmp.path()).await;
12929            if !registered {
12930                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12931            }
12932            let marker = tmp.path().join("started");
12933            let release = tmp.path().join("release");
12934            let mut implementer = spec("implement");
12935            implementer.command = vec![
12936                "sh".into(),
12937                "-c".into(),
12938                format!(
12939                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12940                    marker.display(),
12941                    release.display()
12942                ),
12943            ];
12944            runner.roles.implementers = vec![implementer.clone()];
12945            runner.state.config.agents = vec![implementer];
12946            let pause = runner.pause.clone();
12947            let (result, ()) = tokio::join!(runner.implement(), async {
12948                tokio::time::timeout(Duration::from_secs(10), async {
12949                    while !marker.exists() {
12950                        tokio::time::sleep(Duration::from_millis(10)).await;
12951                    }
12952                })
12953                .await
12954                .unwrap();
12955                pause.park();
12956                std::fs::write(&release, "go").unwrap();
12957            });
12958            result.unwrap();
12959            let candidate = &runner.state.candidates[0];
12960            assert!(candidate.commits > 0, "{candidate:?}");
12961            assert!(candidate.summary.contains("saved-result"));
12962            assert_eq!(candidate.files, 1);
12963            let before = candidate.commits;
12964            runner.implement().await.unwrap();
12965            assert_eq!(runner.state.candidates[0].commits, before);
12966        }
12967    }
12968
12969    #[tokio::test]
12970    #[cfg(unix)]
12971    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12972        let tmp = tempfile::tempdir().unwrap();
12973        let mut runner = registered_runner(tmp.path()).await;
12974        let winner = runner.state.candidates[0].clone();
12975        let cache = tmp.path().join("cache");
12976        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12977        runner.state.config.graph.timeout_resource_wait = 0;
12978        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12979        let crate::cache::AcquireOutcome::Acquired(held) =
12980            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12981        else {
12982            panic!("cache must be free")
12983        };
12984        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12985            from_run: "earlier".into(),
12986            generation: 1,
12987            mode: crate::recovery::Mode::VerifyRepair,
12988            reasons: Vec::new(),
12989            reviewed_head: None,
12990            rounds: Vec::new(),
12991            open_findings: Vec::new(),
12992            evidence: None,
12993            criteria_inherited: true,
12994            repair: crate::recovery::Repair::NotStarted,
12995        });
12996        runner.recovery_repair(&winner).await.unwrap();
12997        assert!(matches!(
12998            runner.state.recovery.as_ref().unwrap().repair,
12999            crate::recovery::Repair::WaitingForRestore { .. }
13000        ));
13001        drop(held);
13002        runner.state.resource_block = None;
13003        runner.state.config.graph.timeout_resource_wait = 1;
13004        runner.recovery_repair(&winner).await.unwrap();
13005        assert!(matches!(
13006            runner.state.recovery.as_ref().unwrap().repair,
13007            crate::recovery::Repair::Done { .. }
13008        ));
13009        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13010    }
13011
13012    #[tokio::test]
13013    #[cfg(unix)]
13014    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13015        use std::os::fd::AsRawFd as _;
13016        for graft in [false, true] {
13017            let tmp = tempfile::tempdir().unwrap();
13018            let mut runner = registered_runner(tmp.path()).await;
13019            runner.state.config.graph.timeout_resource_wait = 1;
13020            let winner = runner.state.candidates[0].clone();
13021            let marker = tmp.path().join("fixer-started");
13022            let mut fixer = spec("fixer");
13023            fixer.command = vec![
13024                "sh".into(),
13025                "-c".into(),
13026                format!("echo started >> '{}'; echo done", marker.display()),
13027            ];
13028            runner.roles.fixer = Some(vec![fixer]);
13029            if graft {
13030                let mut loser = winner.clone();
13031                loser.label = 'B';
13032                loser.index = 1;
13033                runner.state.candidates.push(loser);
13034                runner.state.config.graph.graft_runner_up = true;
13035                runner.state.tally = Some(Tally {
13036                    first_choice: BTreeMap::new(),
13037                    borda: BTreeMap::new(),
13038                    winner: winner.label,
13039                    rankings: 1,
13040                    unanimous_initial: true,
13041                    deliberated: false,
13042                    changed_votes: 0,
13043                    unanimous_final: true,
13044                    tie_break: None,
13045                    judges: 1,
13046                    present: 1,
13047                    quorum: 1,
13048                    met_quorum: true,
13049                    uncontested: None,
13050                });
13051                runner.state.judgements.push(crate::run::Judgement {
13052                    judge: 1,
13053                    seat: "judge-1".into(),
13054                    agent: "judge".into(),
13055                    ranking: vec![winner.label, 'B'],
13056                    reasons: BTreeMap::new(),
13057                    confidence: None,
13058                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13059                    order: vec![0, 1],
13060                    failed: None,
13061                    duration_ms: 0,
13062                });
13063            } else {
13064                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13065                    from_run: "earlier".into(),
13066                    generation: 1,
13067                    mode: crate::recovery::Mode::VerifyRepair,
13068                    reasons: Vec::new(),
13069                    reviewed_head: None,
13070                    rounds: Vec::new(),
13071                    open_findings: Vec::new(),
13072                    evidence: None,
13073                    criteria_inherited: true,
13074                    repair: crate::recovery::Repair::NotStarted,
13075                });
13076            }
13077            let reader = crate::repo_lock::SeatLease::for_checkout(
13078                &winner.worktree,
13079                false,
13080                crate::config::test_coordination().as_ref(),
13081            )
13082            .await
13083            .unwrap();
13084            let pause = runner.pause.clone();
13085            let (result, ()) = tokio::join!(
13086                async {
13087                    if graft {
13088                        runner.graft().await
13089                    } else {
13090                        runner.recovery_repair(&winner).await
13091                    }
13092                },
13093                async {
13094                    tokio::time::sleep(Duration::from_millis(100)).await;
13095                    pause.park();
13096                }
13097            );
13098            result.unwrap();
13099            assert!(!marker.exists(), "a blocked seat must not spawn");
13100            // Exercise the persisted state, as a binary replacement/resume does.
13101            runner.state = RunState::load(&runner.state.id).unwrap();
13102            if graft {
13103                assert_eq!(
13104                    runner.state.graft.as_ref().unwrap().outcome,
13105                    GraftOutcome::WaitingForRestore
13106                );
13107            } else {
13108                assert!(matches!(
13109                    runner.state.recovery.as_ref().unwrap().repair,
13110                    crate::recovery::Repair::WaitingForRestore { .. }
13111                ));
13112            }
13113            drop(reader);
13114            runner.pause = Pause::new();
13115            runner.state.resource_block = None;
13116            // Deferred restoration must still respect an external lock owner.
13117            let external =
13118                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13119            assert_eq!(
13120                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13121                0
13122            );
13123            let leftover = winner.worktree.join("leftover");
13124            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13125            let result = if graft {
13126                runner.graft().await
13127            } else {
13128                runner.recovery_repair(&winner).await
13129            };
13130            assert!(
13131                result
13132                    .unwrap_err()
13133                    .is::<crate::repo_lock::ResourceBlocked>()
13134            );
13135            assert!(leftover.exists());
13136            assert!(!marker.exists());
13137            drop(external);
13138            runner.state.resource_block = None;
13139            let result = if graft {
13140                runner.graft().await
13141            } else {
13142                runner.recovery_repair(&winner).await
13143            };
13144            result.unwrap();
13145            assert!(!leftover.exists(), "restore must precede retry");
13146            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13147            if graft {
13148                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13149            } else {
13150                assert!(matches!(
13151                    runner.state.recovery.as_ref().unwrap().repair,
13152                    crate::recovery::Repair::Done { .. }
13153                ));
13154            }
13155        }
13156    }
13157
13158    #[tokio::test]
13159    #[cfg(unix)]
13160    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13161        use std::os::fd::AsRawFd as _;
13162        let tmp = tempfile::tempdir().unwrap();
13163        let mut runner = registered_runner(tmp.path()).await;
13164        runner.state.config.graph.timeout_resource_wait = 1;
13165        let winner = runner.state.candidates[0].clone();
13166        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13167        runner.state.tally = Some(Tally {
13168            first_choice: BTreeMap::new(),
13169            borda: BTreeMap::new(),
13170            winner: winner.label,
13171            rankings: 0,
13172            unanimous_initial: false,
13173            deliberated: false,
13174            changed_votes: 0,
13175            unanimous_final: false,
13176            tie_break: None,
13177            judges: 0,
13178            present: 0,
13179            quorum: 0,
13180            met_quorum: true,
13181            uncontested: Some("fixture".into()),
13182        });
13183        runner.state.graft = Some(GraftRecord {
13184            pre: pre.clone(),
13185            post: None,
13186            lines: None,
13187            limit: None,
13188            agent: None,
13189            outcome: GraftOutcome::Running,
13190            requested: Vec::new(),
13191            adoption: None,
13192            adoption_error: None,
13193        });
13194        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13195        git::commit_all(&winner.worktree, "unfinished graft")
13196            .await
13197            .unwrap();
13198        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13199        let leftover = winner.worktree.join("leftover");
13200        std::fs::write(&leftover, "untrusted edits").unwrap();
13201        let path = runner.state.repo.join(".git/updater.lock");
13202        let external = std::fs::File::create(&path).unwrap();
13203        assert_eq!(
13204            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13205            0
13206        );
13207        assert!(
13208            runner
13209                .graft()
13210                .await
13211                .unwrap_err()
13212                .is::<crate::repo_lock::ResourceBlocked>()
13213        );
13214        assert!(matches!(
13215            runner.state.graft.as_ref().unwrap().outcome,
13216            GraftOutcome::Running
13217        ));
13218        assert_eq!(
13219            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13220            unfinished
13221        );
13222        assert!(leftover.exists());
13223        drop(external);
13224        std::fs::remove_file(&path).unwrap();
13225        std::fs::create_dir(&path).unwrap();
13226        assert!(runner.graft().await.is_err());
13227        assert!(matches!(
13228            runner.state.graft.as_ref().unwrap().outcome,
13229            GraftOutcome::Running
13230        ));
13231        assert_eq!(
13232            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13233            unfinished
13234        );
13235        assert!(leftover.exists());
13236        std::fs::remove_dir(&path).unwrap();
13237        runner.graft().await.unwrap();
13238        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13239        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13240        assert!(!leftover.exists());
13241    }
13242
13243    #[tokio::test]
13244    #[cfg(unix)]
13245    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13246        use std::os::fd::AsRawFd as _;
13247        let tmp = tempfile::tempdir().unwrap();
13248        let mut runner = registered_runner(tmp.path()).await;
13249        runner.state.prep_complete = Some(false);
13250        runner.state.config.graph.timeout_resource_wait = 1;
13251        runner.state.save().unwrap();
13252        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13253        assert_eq!(
13254            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13255            0
13256        );
13257        runner.execute().await.unwrap();
13258        let disk = RunState::load(&runner.state.id).unwrap();
13259        assert!(disk.driver_exited);
13260        assert_eq!(disk.driver_pid, None);
13261        assert_eq!(runner.state.status, RunStatus::Blocked);
13262        assert_eq!(
13263            runner.state.resource_block.as_ref().unwrap().kind,
13264            "repository coordination"
13265        );
13266        assert!(runner.state.prep_failure.is_none());
13267        assert!(runner.state.seats.is_empty());
13268        runner.pause.park();
13269        // Include headroom for Git subprocesses under parallel test load;
13270        // the repository contention deadline itself remains one second.
13271        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13272            .await
13273            .unwrap()
13274            .unwrap();
13275        assert!(runner.state.parked);
13276        assert!(runner.state.prep_failure.is_none());
13277        let disk = RunState::load(&runner.state.id).unwrap();
13278        assert!(disk.driver_exited);
13279        assert_eq!(disk.driver_pid, None);
13280    }
13281
13282    #[tokio::test]
13283    #[cfg(unix)]
13284    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13285        use std::os::fd::AsRawFd as _;
13286        let tmp = tempfile::tempdir().unwrap();
13287        let mut runner = registered_runner(tmp.path()).await;
13288        runner.state.prep_complete = Some(false);
13289        runner.state.config.graph.timeout_resource_wait = 30;
13290        runner.state.save().unwrap();
13291        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13292        assert_eq!(
13293            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13294            0
13295        );
13296        assert!(
13297            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13298                .await
13299                .is_err()
13300        );
13301        let disk = RunState::load(&runner.state.id).unwrap();
13302        assert!(disk.driver_exited);
13303        assert_eq!(disk.driver_pid, None);
13304        assert!(disk.status.resumable());
13305        assert!(runner.state.driver_exited);
13306        assert_eq!(runner.state.driver_pid, None);
13307        runner.state.save().unwrap();
13308        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13309    }
13310
13311    #[tokio::test]
13312    #[cfg(unix)]
13313    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13314        let tmp = tempfile::tempdir().unwrap();
13315        let mut runner = registered_runner(tmp.path()).await;
13316        let winner = runner.state.candidates[0].clone();
13317        let cache = tmp.path().join("cache");
13318        let started = tmp.path().join("pre-gate-started");
13319        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13320        runner.state.config.verify.gate =
13321            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13322        let home = crate::run::home();
13323        let owner =
13324            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13325        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13326            crate::cache::AcquireOutcome::Acquired(g) => g,
13327            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13328        };
13329        let mut gate = Box::pin(runner.run_gate(&winner));
13330        assert!(
13331            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13332                .await
13333                .is_err()
13334        );
13335        assert!(!started.exists(), "pre-gate must wait for the cache too");
13336        // Simulate the wave that already owns the cache entering invoke.
13337        // It must obtain the editor reservation and finish without waiting
13338        // for the gate that is queued behind its cache.
13339        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13340        let editor = crate::repo_lock::SeatLease::checkout_with(
13341            &winner.worktree,
13342            true,
13343            crate::config::test_coordination().as_ref(),
13344            &wait,
13345        )
13346        .await
13347        .unwrap();
13348        drop(editor);
13349        guard.release();
13350        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13351            .await
13352            .unwrap()
13353            .unwrap();
13354        drop(gate);
13355        assert!(outcomes.iter().all(CommandOutcome::ok));
13356        assert!(started.exists());
13357    }
13358
13359    #[tokio::test]
13360    #[cfg(unix)]
13361    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13362        use std::os::fd::AsRawFd as _;
13363
13364        ask_test_home();
13365        let tmp = tempfile::tempdir().unwrap();
13366        let repo = tmp.path().join("repo");
13367        std::fs::create_dir_all(&repo).unwrap();
13368        init_repo(&repo);
13369        let common = repo.join(".git");
13370        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13371        let mut runner = runner_at(RunStatus::Prep);
13372        runner.state.repo = repo.clone();
13373        runner.state.config.coordination = crate::config::test_coordination();
13374        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13375        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13376        runner.state.config.graph.advise = false;
13377        runner.roles.implementers = vec![spec("implement")];
13378        runner.roles.judges = vec![spec("judge")];
13379        runner.prep().await.unwrap();
13380        let winner = runner.state.candidates[0].clone();
13381        // Emulate edits already left by a completed seat. The fixture CLI
13382        // only reports completion; it performs no repository operations.
13383        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13384        let completed = tmp.path().join("seat-completed");
13385        runner.roles.implementers[0].command = vec![
13386            "sh".into(),
13387            "-c".into(),
13388            format!("touch '{}'; echo completed", completed.display()),
13389        ];
13390        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13391        assert_eq!(
13392            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13393            0
13394        );
13395        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13396        let mut implement = Box::pin(runner.implement());
13397        tokio::select! {
13398            result = &mut implement => panic!("rescue must wait: {result:?}"),
13399            result = tokio::time::timeout(Duration::from_secs(10), async {
13400                while !completed.exists() {
13401                    tokio::time::sleep(Duration::from_millis(20)).await;
13402                }
13403            }) => result.unwrap(),
13404        }
13405        assert!(
13406            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13407                .await
13408                .is_err()
13409        );
13410        assert_eq!(
13411            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13412            before
13413        );
13414        assert_eq!(
13415            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13416            0
13417        );
13418        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13419            .await
13420            .unwrap()
13421            .unwrap();
13422        drop(implement);
13423        assert_ne!(
13424            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13425            before
13426        );
13427        assert!(git::is_clean(&winner.worktree).await.unwrap());
13428
13429        let started = tmp.path().join("pre-gate-started");
13430        let proceed = tmp.path().join("pre-gate-proceed");
13431        let checked = tmp.path().join("gate-ran");
13432        runner.state.config.verify.pre_gate = vec![format!(
13433            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13434            started.display(),
13435            proceed.display()
13436        )];
13437        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13438        assert_eq!(
13439            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13440            0
13441        );
13442        let mut gate = Box::pin(runner.run_gate(&winner));
13443        assert!(
13444            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13445                .await
13446                .is_err()
13447        );
13448        assert!(!started.exists());
13449        assert!(!checked.exists());
13450        assert!(git::is_clean(&winner.worktree).await.unwrap());
13451        assert_eq!(
13452            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13453            0
13454        );
13455        tokio::select! {
13456            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13457            result = tokio::time::timeout(Duration::from_secs(10), async {
13458                while !started.exists() {
13459                    tokio::time::sleep(Duration::from_millis(20)).await;
13460                }
13461            }) => result.unwrap(),
13462        }
13463        assert_eq!(
13464            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13465            -1,
13466            "mechanical edits must hold the real lock"
13467        );
13468        std::fs::write(proceed, "ready").unwrap();
13469        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13470            .await
13471            .unwrap()
13472            .unwrap();
13473        drop(gate);
13474        assert!(outcomes.iter().all(CommandOutcome::ok));
13475        assert!(checked.exists());
13476        assert!(runner.state.pre_gate_commit.is_some());
13477        assert!(git::is_clean(&winner.worktree).await.unwrap());
13478        assert_eq!(
13479            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13480            0,
13481            "gate completion must release coordination"
13482        );
13483        drop(outsider);
13484        std::fs::remove_file(common.join("updater.lock")).unwrap();
13485        std::fs::create_dir(common.join("updater.lock")).unwrap();
13486        std::fs::remove_file(&started).unwrap();
13487        std::fs::remove_file(&checked).unwrap();
13488        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13489        let error = runner.run_gate(&winner).await.unwrap_err();
13490        assert!(
13491            error
13492                .to_string()
13493                .contains("acquire mandatory repository lock")
13494        );
13495        assert!(
13496            !started.exists(),
13497            "failed acquisition must prevent pre-gate"
13498        );
13499        assert!(!checked.exists(), "failed acquisition must prevent gate");
13500        assert!(git::is_clean(&winner.worktree).await.unwrap());
13501        assert_eq!(
13502            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13503            head
13504        );
13505    }
13506
13507    #[tokio::test]
13508    #[cfg(unix)]
13509    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13510        use std::os::fd::AsRawFd as _;
13511        use std::time::Duration;
13512
13513        ask_test_home();
13514        let tmp = tempfile::tempdir().unwrap();
13515        let repo = tmp.path().join("repo");
13516        std::fs::create_dir_all(&repo).unwrap();
13517        init_repo(&repo);
13518        let common = repo.join(".git");
13519        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13520        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13521        assert_eq!(
13522            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13523            0
13524        );
13525        let mut runner = runner_at(RunStatus::Prep);
13526        runner.state.repo = repo.clone();
13527        runner.state.config.coordination = crate::config::test_coordination();
13528        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13529        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13530        runner.state.config.graph.advise = true;
13531        runner.state.config.graph.advisors = 1;
13532        runner.roles.implementers = vec![spec("implement")];
13533        runner.roles.judges = vec![spec("judge")];
13534        let root = runner.state.worktree_root();
13535        let started = tmp.path().join("setup-started");
13536        let proceed = tmp.path().join("setup-proceed");
13537        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13538            run: Some(format!(
13539                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13540                started.display(),
13541                proceed.display()
13542            )),
13543            ..Default::default()
13544        }];
13545        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13546        let config = git::git(&repo, &["config", "--local", "--list"])
13547            .await
13548            .unwrap();
13549        let mut prep = Box::pin(runner.prep());
13550        assert!(
13551            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13552                .await
13553                .is_err()
13554        );
13555        assert!(
13556            !root.exists(),
13557            "contention must prevent even checkout creation"
13558        );
13559        assert!(!started.exists(), "contention must prevent setup");
13560        assert_eq!(
13561            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13562            branches
13563        );
13564        assert_eq!(
13565            git::git(&repo, &["config", "--local", "--list"])
13566                .await
13567                .unwrap(),
13568            config
13569        );
13570        drop(lock);
13571        tokio::select! {
13572            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13573            result = tokio::time::timeout(Duration::from_secs(10), async {
13574                while !started.exists() {
13575                    tokio::time::sleep(Duration::from_millis(20)).await;
13576                }
13577            }) => result.unwrap(),
13578        }
13579        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13580        assert_eq!(
13581            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13582            -1
13583        );
13584        assert!(
13585            tokio::time::timeout(
13586                Duration::from_millis(100),
13587                crate::repo_lock::SeatLease::for_checkout(
13588                    &repo,
13589                    true,
13590                    crate::config::test_coordination().as_ref()
13591                )
13592            )
13593            .await
13594            .is_err(),
13595            "preparation must also exclude local editors"
13596        );
13597        std::fs::write(proceed, "ready").unwrap();
13598        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13599            .await
13600            .unwrap()
13601            .unwrap();
13602        drop(prep);
13603        assert_eq!(runner.state.prep_complete, Some(true));
13604        assert!(root.join("judge-1").exists());
13605        assert!(root.join("advisor-1").exists());
13606        assert_eq!(
13607            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13608            0
13609        );
13610    }
13611
13612    #[tokio::test]
13613    #[cfg(unix)]
13614    async fn preparation_lock_failure_prevents_repository_changes() {
13615        ask_test_home();
13616        let tmp = tempfile::tempdir().unwrap();
13617        let repo = tmp.path().join("repo");
13618        std::fs::create_dir_all(&repo).unwrap();
13619        init_repo(&repo);
13620        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13621        // A directory at the required lock path makes opening the real lock
13622        // fail, rather than merely exercising an invalid registration.
13623        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13624        let mut runner = runner_at(RunStatus::Prep);
13625        runner.state.repo = repo.clone();
13626        runner.state.config.coordination = crate::config::test_coordination();
13627        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13628        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13629        runner.roles.implementers = vec![spec("implement")];
13630        let error = runner.prep().await.unwrap_err();
13631        assert!(
13632            error
13633                .to_string()
13634                .contains("acquire mandatory repository lock")
13635        );
13636        assert!(!runner.state.worktree_root().exists());
13637        assert!(runner.state.candidates.is_empty());
13638        assert!(!repo.join(".git/config.worktree").exists());
13639        assert_eq!(
13640            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13641            "* main"
13642        );
13643    }
13644
13645    #[tokio::test]
13646    async fn preparation_records_creation_intent_before_git_can_fail() {
13647        ask_test_home();
13648        let tmp = tempfile::tempdir().unwrap();
13649        let repo = tmp.path().join("repo");
13650        std::fs::create_dir_all(&repo).unwrap();
13651        init_repo(&repo);
13652        let mut runner = runner_at(RunStatus::Prep);
13653        runner.state.repo = repo.clone();
13654        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13655        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13656        runner.state.config.blind.commit_msg_hook = false;
13657        runner.roles.implementers = vec![spec("one")];
13658        let label = blind::assign_labels(1, runner.state.seed)[0];
13659        let branch = runner.state.branch_for(label);
13660        // A conflicting branch makes Git refuse creation before a candidate
13661        // can be registered. The durable intent must still identify it.
13662        git::git(&repo, &["branch", &branch]).await.unwrap();
13663        assert!(runner.prep().await.is_err());
13664        let saved = RunState::load(&runner.state.id).unwrap();
13665        assert!(saved.candidates.is_empty());
13666        let event = saved
13667            .events
13668            .iter()
13669            .find(|e| e.node == "prep-intent")
13670            .unwrap();
13671        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13672        assert_eq!(intent["branch"], branch);
13673        assert_eq!(intent["start"], runner.state.base_commit);
13674        assert_eq!(
13675            intent["worktree"],
13676            runner
13677                .state
13678                .worktree_root()
13679                .join(format!("cand-{label}"))
13680                .to_string_lossy()
13681                .as_ref()
13682        );
13683        assert_eq!(intent["picks"], serde_json::json!([]));
13684    }
13685
13686    #[tokio::test]
13687    async fn invalid_pinned_sources_create_no_partial_worktrees() {
13688        ask_test_home();
13689        let tmp = tempfile::tempdir().unwrap();
13690        let repo = tmp.path().join("repo");
13691        std::fs::create_dir_all(&repo).unwrap();
13692        init_repo(&repo);
13693        let mut runner = runner_at(RunStatus::Prep);
13694        runner.state.repo = repo.clone();
13695        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13696        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13697        runner.state.sources.pick_commits.push(refs::Input {
13698            token: "missing".into(),
13699            sha: "0".repeat(40),
13700            provenance: refs::Provenance::ExplicitCli,
13701        });
13702        assert!(runner.prep().await.is_err());
13703        assert!(runner.state.candidates.is_empty());
13704        assert!(!runner.state.worktree_root().exists());
13705    }
13706
13707    #[tokio::test]
13708    async fn prose_and_legacy_inference_never_import_into_candidates() {
13709        ask_test_home();
13710        let tmp = tempfile::tempdir().unwrap();
13711        let repo = tmp.path().join("repo");
13712        std::fs::create_dir_all(&repo).unwrap();
13713        init_repo(&repo);
13714        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13715        git::git(&repo, &["checkout", "-b", "magi/history/A"])
13716            .await
13717            .unwrap();
13718        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13719        git::git(&repo, &["add", "."]).await.unwrap();
13720        git::git(&repo, &["commit", "-m", "historical implementation"])
13721            .await
13722            .unwrap();
13723        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13724        git::git(&repo, &["checkout", "main"]).await.unwrap();
13725        for instruction in [
13726            format!("Example: magi/history/A at {source}"),
13727            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13728            format!("Never import magi/history/A or {source}"),
13729            format!("Conversation answer: I mentioned {source} only as background"),
13730        ] {
13731            let mut runner = runner_at(RunStatus::Prep);
13732            runner.state.repo = repo.clone();
13733            runner.state.base_commit = base.clone();
13734            runner.state.instruction = instruction.clone();
13735            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13736            runner.state.config.blind.commit_msg_hook = false;
13737            runner.state.seeds = vec![refs::Seed {
13738                token: "magi/history/A".into(),
13739                sha: source.clone(),
13740                kind: refs::SeedKind::Unmerged,
13741                branch: true,
13742                detail: String::new(),
13743            }];
13744            runner.roles.implementers = vec![spec("one")];
13745            assert_eq!(seeded_instruction(&runner.state), instruction);
13746            // Model an old record with inferred seeds but no structured input field.
13747            let mut legacy = serde_json::to_value(&runner.state).unwrap();
13748            legacy.as_object_mut().unwrap().remove("sources");
13749            runner.state = serde_json::from_value(legacy).unwrap();
13750            assert_eq!(runner.state.sources, refs::Sources::default());
13751            runner.state.save().unwrap();
13752            runner.state = RunState::load(&runner.state.id).unwrap();
13753            runner.prep().await.unwrap();
13754            for candidate in &runner.state.candidates {
13755                assert_eq!(
13756                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13757                    base
13758                );
13759                assert!(!candidate.worktree.join("unwanted").exists());
13760            }
13761        }
13762    }
13763
13764    #[tokio::test]
13765    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13766        ask_test_home();
13767        let tmp = tempfile::tempdir().unwrap();
13768        let repo = tmp.path().join("repo");
13769        std::fs::create_dir_all(&repo).unwrap();
13770        init_repo(&repo);
13771        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13772        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13773        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13774        git::git(&repo, &["add", "."]).await.unwrap();
13775        git::git(&repo, &["commit", "-m", "required seed"])
13776            .await
13777            .unwrap();
13778        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13779        git::git(&repo, &["checkout", "main"]).await.unwrap();
13780        let mut runner = runner_at(RunStatus::Prep);
13781        runner.state.repo = repo.clone();
13782        runner.state.base_commit = base.clone();
13783        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13784        runner.state.config.graph.advise = false;
13785        runner.state.config.blind.commit_msg_hook = false;
13786        runner.state.sources =
13787            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13788                .await
13789                .unwrap();
13790        runner.roles.implementers = vec![spec("one"), spec("two")];
13791        runner.roles.judges = vec![spec("judge")];
13792        let label = blind::assign_labels(2, runner.state.seed)[0];
13793        let branch = runner.state.branch_for(label);
13794        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13795        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13796            .await
13797            .unwrap();
13798        runner.state.candidates.push(Candidate {
13799            index: 0,
13800            label,
13801            agent: "one".into(),
13802            branch,
13803            worktree,
13804            summary: String::new(),
13805            stat: String::new(),
13806            files: 0,
13807            commits: 0,
13808            empty: false,
13809            failed: None,
13810            verified_noop: None,
13811            duration_ms: 0,
13812            folded: false,
13813        });
13814        runner.state.save().unwrap();
13815        // Reload the checkpoint immediately before its first cherry-pick.
13816        runner.state = RunState::load(&runner.state.id).unwrap();
13817        assert_eq!(runner.state.prep_complete, Some(false));
13818        runner.prep().await.unwrap();
13819        assert_eq!(runner.state.prep_complete, Some(true));
13820        assert_eq!(runner.state.candidates.len(), 2);
13821        for candidate in &runner.state.candidates {
13822            assert_eq!(
13823                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13824                git::tree_of(&repo, &seed).await.unwrap()
13825            );
13826            assert_eq!(
13827                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13828                "required commit"
13829            );
13830        }
13831        assert!(runner.state.worktree_root().join("judge-1").exists());
13832        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13833        runner.prep().await.unwrap();
13834        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13835
13836        runner.state.prep_complete = Some(false);
13837        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13838            run: Some(format!(
13839                "test -f '{}'",
13840                tmp.path().join("dependency-ready").display()
13841            )),
13842            ..Default::default()
13843        }];
13844        assert!(runner.prep().await.is_err());
13845        assert!(runner.state.prep_failure.is_some());
13846        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13847        runner.state = RunState::load(&runner.state.id).unwrap();
13848        assert!(
13849            runner.prep().await.is_err(),
13850            "correcting the cause alone must not auto-retry"
13851        );
13852        let failed_at = runner.state.events.last().unwrap().at;
13853        assert!(
13854            runner.retry_preparation(failed_at).is_err(),
13855            "old approvals cannot unlock the hold"
13856        );
13857        let mut task = crate::queue::Task::new(
13858            "retry".into(),
13859            "retry setup".into(),
13860            repo.clone(),
13861            crate::queue::Source::Human,
13862        );
13863        task.start(runner.state.id.clone());
13864        task.hold_machine(Some("setup failed".into()));
13865        task.release_operator("cli", None);
13866        runner
13867            .retry_preparation(task.resumption.as_ref().unwrap().at)
13868            .unwrap();
13869        runner.prep().await.unwrap();
13870        assert!(runner.state.prep_failure.is_none());
13871        assert_eq!(runner.state.prep_complete, Some(true));
13872        assert_eq!(runner.state.candidates.len(), 2);
13873        assert!(runner.state.worktree_root().join("judge-1").exists());
13874        for candidate in &runner.state.candidates {
13875            assert_eq!(
13876                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13877                git::tree_of(&repo, &seed).await.unwrap()
13878            );
13879        }
13880    }
13881
13882    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13883        let mut heads = Vec::new();
13884        for candidate in candidates {
13885            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13886        }
13887        heads
13888    }
13889
13890    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13891        ask_test_home();
13892        let tmp = tempfile::tempdir().unwrap();
13893        let repo = tmp.path().join("repo");
13894        std::fs::create_dir(&repo).unwrap();
13895        init_repo(&repo);
13896        let mut runner = runner_at(RunStatus::Reviewing);
13897        runner.state.repo = repo.clone();
13898        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13899        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13900        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13901            let branch = format!("candidate-{label}");
13902            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13903            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13904                .await
13905                .unwrap();
13906            if label == 'B' {
13907                std::fs::write(worktree.join("change"), "winner").unwrap();
13908                git::rescue_commit(&worktree, "candidate change")
13909                    .await
13910                    .unwrap();
13911            }
13912            runner.state.candidates.push(Candidate {
13913                index,
13914                label,
13915                agent: "alpha".into(),
13916                branch,
13917                worktree,
13918                summary: "completed reply".into(),
13919                stat: String::new(),
13920                files: usize::from(label == 'B'),
13921                commits: usize::from(label == 'B'),
13922                empty: label == 'A',
13923                failed: None,
13924                verified_noop: None,
13925                duration_ms: 1,
13926                folded: false,
13927            });
13928        }
13929        runner.state.judge_skipped = true;
13930        runner.state.tally = Some(Tally {
13931            first_choice: BTreeMap::new(),
13932            borda: BTreeMap::new(),
13933            winner: 'B',
13934            rankings: 0,
13935            unanimous_initial: false,
13936            deliberated: false,
13937            changed_votes: 0,
13938            unanimous_final: false,
13939            tie_break: None,
13940            judges: 0,
13941            present: 0,
13942            quorum: 0,
13943            met_quorum: true,
13944            uncontested: Some("only B produced a change".into()),
13945        });
13946        runner.state.save().unwrap();
13947        (tmp, runner)
13948    }
13949
13950    #[tokio::test]
13951    #[cfg(unix)]
13952    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13953        for (solo, failed_checkpoint) in
13954            [(true, false), (false, false), (true, true), (false, true)]
13955        {
13956            let (_tmp, mut runner) = preservation_resume_fixture().await;
13957            if solo {
13958                runner.state.candidates.retain(|c| c.label == 'A');
13959            }
13960            runner.state.candidates[0].empty = false;
13961            runner.state.candidates[0].summary.clear();
13962            runner.state.tally = None;
13963            runner.state.judge_skipped = false;
13964            runner.state.status = RunStatus::Implementing;
13965            runner.state.config.disk.min_free_bytes = u64::MAX;
13966            let mut implementer = spec("alpha");
13967            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13968            runner.roles.implementers = vec![implementer];
13969            let patch = runner.state.dir().join("artifacts/cand-A.patch");
13970            if failed_checkpoint {
13971                std::fs::create_dir_all(&patch).unwrap();
13972            }
13973            runner.state.save().unwrap();
13974            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13975            runner.state = RunState::load(&runner.state.id).unwrap();
13976            assert_eq!(runner.state.status, RunStatus::Blocked);
13977            assert!(runner.state.interrupted_implementations.contains(&'A'));
13978            assert_eq!(runner.state.candidates[0].commits, 0);
13979            assert!(!runner.state.candidates[0].empty);
13980            assert!(runner.state.candidates[0].failed.is_none());
13981            if failed_checkpoint {
13982                assert!(runner.state.candidate_preservation.contains_key(&'A'));
13983                std::fs::remove_dir(patch).unwrap();
13984            }
13985            runner.state.resource_block = None;
13986            runner.state.config.disk.min_free_bytes = 0;
13987            runner.roles.implementers[0].command = vec![
13988                "sh".into(),
13989                "-c".into(),
13990                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13991            ];
13992            runner.implement().await.unwrap();
13993            assert!(runner.state.interrupted_implementations.is_empty());
13994            assert!(runner.state.candidate_preservation.is_empty());
13995            assert_eq!(runner.state.candidates[0].commits, 1);
13996            assert!(!runner.state.candidates[0].empty);
13997            assert!(
13998                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13999                    .unwrap()
14000                    .contains("retried empty interrupted seat")
14001            );
14002            if !solo {
14003                assert_eq!(runner.state.candidates[1].commits, 1);
14004            }
14005        }
14006    }
14007
14008    #[tokio::test]
14009    #[cfg(unix)]
14010    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14011        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14012            let (_tmp, mut runner) = preservation_resume_fixture().await;
14013            runner.state.candidates.retain(|c| c.label == 'A');
14014            runner.state.candidates[0].empty = false;
14015            runner.state.candidates[0].summary.clear();
14016            runner.state.tally = None;
14017            runner.state.judge_skipped = false;
14018            runner.state.status = RunStatus::Implementing;
14019            // Force the real post-spawn watchdog, independently of available
14020            // disk space. Dirty/committed work predates its first measurement.
14021            runner.state.config.disk.min_free_bytes = u64::MAX;
14022            let wt = runner.state.candidates[0].worktree.clone();
14023            std::fs::write(
14024                wt.join("interrupted-source"),
14025                "retain implementation work\n",
14026            )
14027            .unwrap();
14028            if committed {
14029                git::rescue_commit(&wt, "implementation checkpoint")
14030                    .await
14031                    .unwrap();
14032            }
14033            let lock = git::git(
14034                &wt,
14035                &[
14036                    "rev-parse",
14037                    "--path-format=absolute",
14038                    "--git-path",
14039                    "index.lock",
14040                ],
14041            )
14042            .await
14043            .unwrap();
14044            if block_rescue {
14045                std::fs::write(&lock, "fixture-owned lock").unwrap();
14046            }
14047            let mut implementer = spec("alpha");
14048            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14049            runner.roles.implementers = vec![implementer];
14050            runner.state.save().unwrap();
14051            let result = runner.implement().await;
14052            assert_eq!(result.is_err(), block_rescue);
14053            assert_eq!(
14054                runner.state.resource_block.as_ref().unwrap().kind,
14055                "disk pressure"
14056            );
14057            assert!(
14058                runner
14059                    .state
14060                    .events
14061                    .iter()
14062                    .any(|e| e.message.contains("agent execution interrupted"))
14063            );
14064            assert!(
14065                runner
14066                    .state
14067                    .dir()
14068                    .join("artifacts/candidate-A-recovery/staged.patch")
14069                    .exists()
14070            );
14071            if block_rescue {
14072                let saved = RunState::load(&runner.state.id).unwrap();
14073                assert!(saved.candidate_preservation.contains_key(&'A'));
14074                assert!(!saved.candidates[0].empty);
14075                assert_eq!(
14076                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14077                    "retain implementation work\n"
14078                );
14079                std::fs::remove_file(lock).unwrap();
14080                runner.state = saved;
14081            } else {
14082                assert_eq!(runner.state.status, RunStatus::Blocked);
14083                assert_eq!(runner.state.candidates[0].commits, 1);
14084            }
14085            // Recovery must save the candidate before deciding whether another
14086            // implementation is needed. No implementer is available on retry.
14087            runner.roles.implementers.clear();
14088            runner.state.config.disk.min_free_bytes = 0;
14089            runner.state.resource_block = None;
14090            runner.implement().await.unwrap();
14091            assert!(runner.state.candidate_preservation.is_empty());
14092            assert_eq!(runner.state.candidates[0].commits, 1);
14093            assert!(!runner.state.candidates[0].empty);
14094            assert!(git::is_clean(&wt).await.unwrap());
14095            assert!(
14096                crate::run::read_artifact(&runner.state, "cand-A.patch")
14097                    .unwrap()
14098                    .contains("retain implementation work")
14099            );
14100        }
14101    }
14102
14103    #[tokio::test]
14104    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14105        let (_tmp, mut runner) = preservation_resume_fixture().await;
14106        // Schema 15 already confirmed the unfolded no-change result.
14107        runner.implement().await.unwrap();
14108        assert!(runner.state.candidate_preservation.is_empty());
14109        assert!(
14110            !runner
14111                .state
14112                .dir()
14113                .join("artifacts/candidate-A-recovery")
14114                .exists()
14115        );
14116        runner.fold_losers().await.unwrap();
14117        assert!(!runner.state.candidates[0].worktree.exists());
14118        for schema in [14, crate::run::SCHEMA] {
14119            runner.state.schema = schema;
14120            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14121            runner.state.save().unwrap();
14122            runner.state = RunState::load(&runner.state.id).unwrap();
14123            runner.implement().await.unwrap();
14124            assert!(runner.state.candidate_preservation.is_empty());
14125            assert!(runner.state.candidates[0].folded);
14126            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14127        }
14128    }
14129
14130    #[tokio::test]
14131    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14132        let (_tmp, mut runner) = preservation_resume_fixture().await;
14133        let wt = runner.state.candidates[0].worktree.clone();
14134        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14135        runner.state.schema = 14;
14136        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14137        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14138        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14139        for _ in 0..2 {
14140            assert!(runner.implement().await.is_err());
14141            assert_eq!(runner.state.status, RunStatus::Blocked);
14142            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14143            assert!(
14144                runner.state.candidates[0].empty,
14145                "old verdict must remain unusable"
14146            );
14147            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14148            assert!(runner.state.judge_skipped);
14149            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14150            assert!(
14151                crate::run::read_artifact(&runner.state, "cand-A.patch")
14152                    .unwrap()
14153                    .contains("owned edits")
14154            );
14155            assert!(wt.exists());
14156            assert_eq!(
14157                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14158                    .await
14159                    .unwrap(),
14160                1
14161            );
14162            runner.state = RunState::load(&runner.state.id).unwrap();
14163        }
14164    }
14165
14166    #[tokio::test]
14167    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14168        let (_tmp, mut runner) = preservation_resume_fixture().await;
14169        let wt = runner.state.candidates[0].worktree.clone();
14170        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14171        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14172        let index_lock = PathBuf::from(
14173            git::git(
14174                &wt,
14175                &[
14176                    "rev-parse",
14177                    "--path-format=absolute",
14178                    "--git-path",
14179                    "index.lock",
14180                ],
14181            )
14182            .await
14183            .unwrap(),
14184        );
14185        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14186        runner.state.schema = 14;
14187        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14188        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14189        assert!(runner.implement().await.is_err());
14190        assert!(runner.after_implement().is_err());
14191        assert!(runner.fold_losers().await.is_err());
14192        assert_eq!(
14193            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14194            "failed seat edits"
14195        );
14196        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14197        runner.state = RunState::load(&runner.state.id).unwrap();
14198        std::fs::remove_file(index_lock).unwrap();
14199        // Recovery finds changes that the old uncontested result never judged.
14200        assert!(runner.implement().await.is_err());
14201        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14202        assert!(runner.state.candidates[0].empty);
14203        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14204        assert!(
14205            crate::run::read_artifact(&runner.state, "cand-A.patch")
14206                .unwrap()
14207                .contains("failed seat edits")
14208        );
14209        assert!(wt.exists());
14210    }
14211
14212    #[tokio::test]
14213    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14214        let (_tmp, mut runner) = preservation_resume_fixture().await;
14215        let repo = runner.state.repo.clone();
14216        let common = git::git(
14217            &repo,
14218            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14219        )
14220        .await
14221        .unwrap();
14222        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14223        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14224        assert!(runner.fold_losers().await.is_err());
14225        assert!(!runner.state.candidates[0].worktree.exists());
14226        assert!(!runner.state.candidates[0].folded);
14227        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14228        runner.state = RunState::load(&runner.state.id).unwrap();
14229        std::fs::remove_file(branch_lock).unwrap();
14230        let judge = runner.state.worktree_root().join("judge-1");
14231        git::worktree_add_detached(&repo, &judge, "main")
14232            .await
14233            .unwrap();
14234        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14235        runner.roles.judges = vec![spec("judge")];
14236        assert!(runner.fold_losers().await.is_err());
14237        runner.state = RunState::load(&runner.state.id).unwrap();
14238        assert!(
14239            runner.state.candidates[0].folded,
14240            "candidate progress saved before judge failure"
14241        );
14242        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14243        assert!(judge.join("uncommitted").exists());
14244        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14245        runner.fold_losers().await.unwrap();
14246        assert!(!judge.exists());
14247        runner.fold_losers().await.unwrap();
14248    }
14249
14250    #[tokio::test]
14251    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14252        let (_tmp, mut runner) = preservation_resume_fixture().await;
14253        runner.state.candidates.retain(|c| c.label == 'B');
14254        runner.state.tally = None;
14255        runner.state.judge_skipped = false;
14256        runner.state.prep_complete = Some(true);
14257        runner.state.config.graph.advise = false;
14258        runner.state.config.graph.selected_implementer = Some("alpha".into());
14259        let wt = runner.state.candidates[0].worktree.clone();
14260        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14261        runner
14262            .state
14263            .candidate_preservation
14264            .insert('B', "awaiting persistence".into());
14265        let index_lock = git::git(
14266            &wt,
14267            &[
14268                "rev-parse",
14269                "--path-format=absolute",
14270                "--git-path",
14271                "index.lock",
14272            ],
14273        )
14274        .await
14275        .unwrap();
14276        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14277        runner.state.save().unwrap();
14278        assert!(runner.execute_graph().await.is_err());
14279        assert_eq!(runner.state.status, RunStatus::Blocked);
14280        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14281        std::fs::remove_file(index_lock).unwrap();
14282        // Park at the next graph boundary: this tests the real selected-seat
14283        // hold guard without dispatching reviewers or publishing anything.
14284        for _ in 0..2 {
14285            runner.state = RunState::load(&runner.state.id).unwrap();
14286            runner.pause.park();
14287            runner.execute_graph().await.unwrap();
14288            assert_eq!(runner.state.status, RunStatus::Implementing);
14289            assert!(runner.state.candidate_preservation.is_empty());
14290            assert_eq!(runner.state.candidates[0].commits, 2);
14291            assert!(runner.state.tally.is_none());
14292        }
14293        runner.implement().await.unwrap();
14294        assert_eq!(runner.state.status, RunStatus::Judging);
14295        // A genuine selected-implementer hold must still require release.
14296        runner.state.status = RunStatus::Blocked;
14297        runner.state.candidates[0].failed = Some("implementation failed".into());
14298        runner.state.save().unwrap();
14299        let error = runner.execute_graph().await.unwrap_err();
14300        assert!(
14301            error
14302                .to_string()
14303                .contains("selected implementation is held")
14304        );
14305    }
14306
14307    #[tokio::test]
14308    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14309        let (_tmp, mut runner) = preservation_resume_fixture().await;
14310        let repo = runner.state.repo.clone();
14311        let candidate = runner.state.candidates[0].clone();
14312        // Remove only the disposable fixture directory, leaving Git's linked
14313        // worktree registration and branch intact, as in a vanished checkout.
14314        let registered = git::worktree_holding(&repo, &candidate.branch)
14315            .await
14316            .unwrap()
14317            .unwrap();
14318        assert_eq!(
14319            registered.canonicalize().unwrap(),
14320            candidate.worktree.canonicalize().unwrap()
14321        );
14322        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14323        assert_eq!(
14324            git::worktree_holding(&repo, &candidate.branch)
14325                .await
14326                .unwrap(),
14327            Some(registered)
14328        );
14329        assert!(
14330            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14331            "the stale registration initially keeps the branch checked out"
14332        );
14333        runner.fold_losers().await.unwrap();
14334        assert!(runner.state.candidates[0].folded);
14335        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14336        assert_eq!(
14337            git::worktree_holding(&repo, &candidate.branch)
14338                .await
14339                .unwrap(),
14340            None
14341        );
14342        runner.state = RunState::load(&runner.state.id).unwrap();
14343        runner.fold_losers().await.unwrap();
14344    }
14345
14346    #[tokio::test]
14347    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14348        let (_tmp, mut runner) = preservation_resume_fixture().await;
14349        let repo = runner.state.repo.clone();
14350        let candidate = runner.state.candidates[0].clone();
14351        let winner = runner.state.candidates[1].clone();
14352        let registered = git::worktree_holding(&repo, &candidate.branch)
14353            .await
14354            .unwrap()
14355            .unwrap();
14356        assert_eq!(
14357            registered.canonicalize().unwrap(),
14358            candidate.worktree.canonicalize().unwrap()
14359        );
14360        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14361        assert_eq!(
14362            git::worktree_holding(&repo, &candidate.branch)
14363                .await
14364                .unwrap(),
14365            Some(registered)
14366        );
14367        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14368
14369        for _ in 0..2 {
14370            fold_run(&mut runner.state, false, &crate::run::home())
14371                .await
14372                .unwrap();
14373            runner.state = RunState::load(&runner.state.id).unwrap();
14374            assert!(runner.state.candidates[0].folded);
14375            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14376            assert_eq!(
14377                git::worktree_holding(&repo, &candidate.branch)
14378                    .await
14379                    .unwrap(),
14380                None
14381            );
14382            assert!(!runner.state.candidates[1].folded);
14383            assert!(winner.worktree.exists());
14384            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14385        }
14386    }
14387
14388    #[tokio::test]
14389    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14390        ask_test_home();
14391        let tmp = tempfile::tempdir().unwrap();
14392        let repo = tmp.path().join("repo");
14393        std::fs::create_dir(&repo).unwrap();
14394        init_repo(&repo);
14395        let mut runner = runner_at(RunStatus::Implementing);
14396        runner.state.repo = repo.clone();
14397        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14398        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14399        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14400            let branch = format!("candidate-{label}");
14401            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14402            git::worktree_add_branch(&repo, &wt, &branch, "main")
14403                .await
14404                .unwrap();
14405            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14406            git::git(&wt, &["add", "README.md"]).await.unwrap();
14407            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14408            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14409            runner.state.candidates.push(Candidate {
14410                index,
14411                label,
14412                agent: "alpha".to_owned(),
14413                branch,
14414                worktree: wt,
14415                summary: "completed implementation".to_owned(),
14416                stat: String::new(),
14417                files: 0,
14418                commits: 0,
14419                empty: false,
14420                failed: None,
14421                verified_noop: None,
14422                duration_ms: 42,
14423                folded: false,
14424            });
14425            runner
14426                .state
14427                .candidate_preservation
14428                .insert(label, "awaiting persistence".to_owned());
14429        }
14430        let wt = runner.state.candidates[0].worktree.clone();
14431        let index_lock = git::git(
14432            &wt,
14433            &[
14434                "rev-parse",
14435                "--path-format=absolute",
14436                "--git-path",
14437                "index.lock",
14438            ],
14439        )
14440        .await
14441        .unwrap();
14442        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14443        runner.state.save().unwrap();
14444        assert!(runner.persist_candidates().await.is_err());
14445        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14446        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14447        assert_eq!(
14448            runner.state.candidates[1].commits, 1,
14449            "other results are still collected"
14450        );
14451        assert!(
14452            !runner.state.candidates[0].empty,
14453            "failed rescue is not no change"
14454        );
14455        assert_eq!(
14456            runner.state.candidates[0].summary,
14457            "completed implementation"
14458        );
14459        assert!(runner.after_implement().is_err());
14460        let home = tmp.path().join("fold-home");
14461        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14462        assert!(wt.join("new-file").exists());
14463        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14464        assert!(recovery.join("staged.patch").exists());
14465        assert!(recovery.join("unstaged.patch").exists());
14466        assert_eq!(
14467            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14468            "untracked edits\n"
14469        );
14470        runner.state.begin_pass();
14471        runner.state.end_pass_parked(ParkCause::Upgrade {
14472            from: "1.0".into(),
14473            to: Some("1.1".into()),
14474        });
14475        runner.state.save().unwrap();
14476        runner.state = RunState::load(&runner.state.id).unwrap();
14477        assert!(matches!(
14478            runner.state.passes.last().unwrap().end,
14479            Some(crate::run::PassEnd::Parked {
14480                cause: ParkCause::Upgrade { .. }
14481            })
14482        ));
14483        std::fs::remove_file(index_lock).unwrap();
14484        // No implementer is configured: a persistence retry cannot dispatch a seat.
14485        runner.implement().await.unwrap();
14486        assert!(runner.state.candidate_preservation.is_empty());
14487        assert!(
14488            runner
14489                .state
14490                .candidates
14491                .iter()
14492                .all(|c| c.commits == 1 && !c.empty)
14493        );
14494        assert!(git::is_clean(&wt).await.unwrap());
14495        assert!(
14496            crate::run::read_artifact(&runner.state, "cand-A.patch")
14497                .unwrap()
14498                .contains("later tracked edits")
14499        );
14500    }
14501
14502    #[tokio::test]
14503    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14504        ask_test_home();
14505        let tmp = tempfile::tempdir().unwrap();
14506        let repo = tmp.path().join("repo");
14507        std::fs::create_dir(&repo).unwrap();
14508        init_repo(&repo);
14509        let wt = tmp.path().join("candidate");
14510        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14511            .await
14512            .unwrap();
14513        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14514        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14515        assert!(wt.join("uncommitted").exists());
14516        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14517        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14518        let mut runner = runner_at(RunStatus::Implementing);
14519        runner.state.repo = repo.clone();
14520        runner.state.base_commit = "missing-base".to_owned();
14521        runner.state.candidates.push(Candidate {
14522            index: 0,
14523            label: 'A',
14524            agent: "alpha".into(),
14525            branch: "candidate".into(),
14526            worktree: wt.clone(),
14527            summary: "completed".into(),
14528            stat: String::new(),
14529            files: 0,
14530            commits: 0,
14531            empty: false,
14532            failed: None,
14533            verified_noop: None,
14534            duration_ms: 1,
14535            folded: false,
14536        });
14537        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14538        assert!(
14539            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14540                .await
14541                .is_err()
14542        );
14543        assert!(!runner.state.candidates[0].folded);
14544        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14545        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14546        runner
14547            .state
14548            .candidate_preservation
14549            .insert('A', "pending".into());
14550        assert!(runner.persist_candidates().await.is_err());
14551        assert!(!runner.state.candidates[0].empty);
14552        assert!(!runner.state.candidates[0].folded);
14553        assert!(wt.exists());
14554        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14555        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14556        std::fs::create_dir_all(&patch_path).unwrap();
14557        assert!(
14558            runner.persist_candidates().await.is_err(),
14559            "artifact failure stays pending"
14560        );
14561        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14562        assert!(!runner.state.candidates[0].empty);
14563        std::fs::remove_dir(patch_path).unwrap();
14564        runner.persist_candidates().await.unwrap();
14565        assert!(
14566            runner.state.candidates[0].empty,
14567            "only successful inspection confirms no change"
14568        );
14569    }
14570
14571    fn init_repo(dir: &Path) {
14572        let run = |args: &[&str]| {
14573            let out = std::process::Command::new("git")
14574                .args(args)
14575                .current_dir(dir)
14576                .quiet()
14577                .output()
14578                .expect("spawn git");
14579            assert!(
14580                out.status.success(),
14581                "git {args:?} failed: {}",
14582                String::from_utf8_lossy(&out.stderr)
14583            );
14584        };
14585        run(&["init", "-b", "main"]);
14586        run(&["config", "user.name", "magi test"]);
14587        run(&["config", "user.email", "magi@example.com"]);
14588        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14589        run(&["add", "-A"]);
14590        run(&["commit", "-m", "init"]);
14591    }
14592
14593    // `settle_questions` is what closes the ghost the phone showed: a run's
14594    // seat asked something, the run then ended, and nothing was left to
14595    // abandon the question it left `open`. `HOME` is a process-wide
14596    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14597    // first time it runs in the binary — every test below still reaches the
14598    // same directory whichever call won, and each gets its own run id from
14599    // `RunState::new`, so they never collide there.
14600    fn ask_test_home() {
14601        crate::run::pin_test_home();
14602    }
14603
14604    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14605    /// reads nothing else off it.
14606    fn runner_at(status: RunStatus) -> Runner {
14607        let mut state = RunState::new(
14608            PathBuf::from("/nonexistent/repo"),
14609            "main".to_owned(),
14610            "deadbeef".to_owned(),
14611            "task".to_owned(),
14612            Config::default(),
14613        );
14614        // These fixtures share one home and save concurrently. Short clock
14615        // IDs can collide, making an ownership test read another test's run.
14616        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14617        state.id = format!(
14618            "20000101-000000-{:04x}",
14619            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14620        );
14621        state.status = status;
14622        Runner {
14623            state,
14624            roles: ResolvedRoles {
14625                implementers: Vec::new(),
14626                judges: Vec::new(),
14627                reviewers: Vec::new(),
14628                fixer: None,
14629                conductor: conductor(),
14630                implementer_roster: Vec::new(),
14631                judge_roster: Vec::new(),
14632                reviewer_roster: Vec::new(),
14633            },
14634            sem: Arc::new(Semaphore::new(1)),
14635            pause: Pause::new(),
14636            interrupt: Pause::new(),
14637        }
14638    }
14639
14640    #[test]
14641    fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14642        crate::run::pin_test_home();
14643        let mut runner = runner_at(RunStatus::Gating);
14644        runner.state.save().unwrap();
14645        // run.json can commit, but the subsequent lease metadata write fails.
14646        // Cleanup must run after releasing the short ownership lock.
14647        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14648        assert!(runner.claim_driver().is_err());
14649        assert!(runner.state.driver_exited);
14650        assert_eq!(runner.state.driver_pid, None);
14651        let disk = RunState::load(&runner.state.id).unwrap();
14652        assert!(disk.driver_exited);
14653        assert_eq!(disk.driver_pid, None);
14654        assert_eq!(disk.status, RunStatus::Gating);
14655    }
14656
14657    #[tokio::test]
14658    async fn an_early_graph_error_releases_its_driver() {
14659        crate::run::pin_test_home();
14660        let mut runner = runner_at(RunStatus::Blocked);
14661        runner.state.config.graph.selected_implementer = Some("selected".into());
14662        runner.state.save().unwrap();
14663        assert!(runner.execute().await.is_err());
14664        let disk = RunState::load(&runner.state.id).unwrap();
14665        assert!(disk.driver_exited);
14666        assert_eq!(disk.driver_pid, None);
14667        assert_eq!(disk.status, RunStatus::Blocked);
14668    }
14669
14670    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14671        crate::run::MergeRepair {
14672            question: "repair-question".into(),
14673            head: "old-head".into(),
14674            pr: "https://github.com/o/r/pull/42".into(),
14675            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14676            gate: vec![outcome(Some(0), false)],
14677            gate_head: Some("old-head".into()),
14678            gate_fixes: Vec::new(),
14679            pre_gate: Vec::new(),
14680            phase,
14681            failure: None,
14682        }
14683    }
14684
14685    #[tokio::test]
14686    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14687        crate::run::pin_test_home();
14688        let mut runner = runner_at(RunStatus::Landing);
14689        runner.state.merge_repairs.push(merge_repair_request(
14690            crate::run::MergeRepairPhase::Requested,
14691        ));
14692        runner.state.save().unwrap();
14693        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14694        assert!(
14695            runner
14696                .execute()
14697                .await
14698                .unwrap_err()
14699                .to_string()
14700                .contains("already running")
14701        );
14702        assert_eq!(
14703            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14704            crate::run::MergeRepairPhase::Requested
14705        );
14706        drop(claim);
14707        let mut disk = runner.state.clone();
14708        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14709        disk.save().unwrap();
14710        assert!(
14711            runner
14712                .execute()
14713                .await
14714                .unwrap_err()
14715                .to_string()
14716                .contains("decision changed")
14717        );
14718        assert_eq!(
14719            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14720            crate::run::MergeRepairPhase::Complete
14721        );
14722    }
14723
14724    #[test]
14725    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14726        crate::run::pin_test_home();
14727        let mut runner = runner_at(RunStatus::Landing);
14728        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14729        runner.state.reviews = request.reviews.clone();
14730        runner.state.gate = request.gate.clone();
14731        runner.state.gate_ran = true;
14732        runner.state.land_approval = Some(crate::run::LandApproval {
14733            question: "old".into(),
14734            head: "old-head".into(),
14735        });
14736        runner.state.merge_repairs.push(request);
14737        runner.begin_merge_repair_review(0).unwrap();
14738        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14739        assert!(!runner.state.gate_status().ok());
14740        assert!(runner.state.land_approval.is_none());
14741        let disk = RunState::load(&runner.state.id).unwrap();
14742        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14743        assert!(disk.merge_repairs[0].gate[0].ok());
14744        assert_eq!(
14745            disk.merge_repairs[0].phase,
14746            crate::run::MergeRepairPhase::Reviewing
14747        );
14748        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14749        assert!(disk.reviews.is_empty());
14750    }
14751
14752    #[tokio::test]
14753    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14754        crate::run::pin_test_home();
14755        for phase in [
14756            crate::run::MergeRepairPhase::Fixing,
14757            crate::run::MergeRepairPhase::Requested,
14758        ] {
14759            let mut runner = runner_at(RunStatus::Landing);
14760            runner.state.config.graph.land_rounds = 0;
14761            runner.state.merge_repairs.push(merge_repair_request(phase));
14762            runner.state.save().unwrap();
14763            assert!(!runner.repair_merge_review().await.unwrap());
14764            assert_eq!(runner.state.status, RunStatus::Blocked);
14765            assert_eq!(
14766                runner.state.merge_repairs[0].phase,
14767                crate::run::MergeRepairPhase::Failed
14768            );
14769            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14770            assert!(
14771                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14772                    "interrupted"
14773                } else {
14774                    "budget exhausted"
14775                })
14776            );
14777            assert!(runner.state.merge.is_none());
14778        }
14779    }
14780
14781    /// `park_here` folding in the reason `Pause::park_because` recorded -
14782    /// this is what lets an operator reading a run's events tell an
14783    /// interrupt-driven park from an ordinary shutdown park.
14784    #[test]
14785    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14786        crate::run::pin_test_home();
14787        let mut runner = runner_at(RunStatus::Implementing);
14788        let interrupt = Pause::new();
14789        runner.watch_interrupt(interrupt.clone());
14790
14791        interrupt.park_because("task a1b2 asked to run first");
14792
14793        assert!(runner.park_here().expect("park_here"));
14794        assert!(runner.state.parked);
14795        let last = runner.state.events.last().expect("a park event");
14796        assert_eq!(last.node, "park");
14797        assert!(
14798            last.message.contains("task a1b2 asked to run first"),
14799            "expected the interrupt reason in {:?}",
14800            last.message
14801        );
14802    }
14803
14804    /// The cause is stored on the pass when it parks, and the first cause a
14805    /// shared pause was given is the one that stays.
14806    #[test]
14807    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14808        crate::run::pin_test_home();
14809        let mut runner = runner_at(RunStatus::Implementing);
14810        runner.state.begin_pass();
14811        let shutdown = Pause::new();
14812        runner.on_pause(shutdown.clone());
14813        let upgrade = ParkCause::Upgrade {
14814            from: "0.121.0".to_owned(),
14815            to: Some("0.121.1".to_owned()),
14816        };
14817        shutdown.park_for(upgrade.clone());
14818        shutdown.park_for(ParkCause::Operator);
14819
14820        assert!(runner.park_here().expect("park_here"));
14821        assert_eq!(
14822            runner.state.passes[0].end,
14823            Some(crate::run::PassEnd::Parked { cause: upgrade })
14824        );
14825        let last = runner.state.events.last().expect("a park event");
14826        assert!(last.message.contains("0.121.0"), "{}", last.message);
14827    }
14828
14829    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14830    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14831    /// cleared) must not make a *different* run - one only watching its own,
14832    /// unshared interrupt `Pause` - see itself as parked. If a future change
14833    /// ever collapsed these back into one handle, the interrupt scheduler
14834    /// would park every run for the rest of the daemon's life, not just the
14835    /// one it meant to interrupt.
14836    #[test]
14837    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14838        crate::run::pin_test_home();
14839        let mut runner = runner_at(RunStatus::Implementing);
14840        let shutdown = Pause::new();
14841        runner.on_pause(shutdown.clone());
14842        let interrupt = Pause::new();
14843        runner.watch_interrupt(interrupt.clone());
14844
14845        // Nobody has asked for anything yet.
14846        assert!(!runner.park_here().expect("park_here"));
14847        assert!(!runner.state.parked);
14848
14849        // Only the interrupt handle fires; the shutdown handle stays clear.
14850        interrupt.park_because("test");
14851        assert!(!shutdown.parked());
14852        assert!(runner.park_here().expect("park_here"));
14853    }
14854
14855    /// The property every prior attempt at this feature failed to pin down:
14856    /// asking a run to park while one of its nodes has a real, in-flight
14857    /// async operation running (an agent call, in production) must not cut
14858    /// that operation short. `park_here` is only ever consulted *between*
14859    /// `execute`'s node calls - see its own doc - so nothing inside a node
14860    /// can observe a park request until the node itself returns. This proves
14861    /// that structurally, with real `tokio` concurrency and a channel
14862    /// handshake (never a sleep, which would only prove "usually", not
14863    /// "cannot"): the "node" below reports that it has genuinely started,
14864    /// and only then is the park requested; the node still has to be told to
14865    /// finish before `park_here` is ever called, exactly mirroring every
14866    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14867    /// in `execute`.
14868    #[tokio::test]
14869    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14870        crate::run::pin_test_home();
14871        let mut runner = runner_at(RunStatus::Implementing);
14872        let interrupt = Pause::new();
14873        runner.watch_interrupt(interrupt.clone());
14874
14875        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14876        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14877
14878        // Stands in for one node's in-flight agent call: it proves it has
14879        // genuinely started, then blocks - exactly as a spawned CLI process
14880        // does - until told to finish.
14881        let node = async move {
14882            started_tx.send(()).expect("send started");
14883            finish_rx.await.expect("recv finish");
14884            "node finished"
14885        };
14886
14887        let interrupter = async move {
14888            started_rx.await.expect("recv started");
14889            // The call is now genuinely in flight. Ask it to park.
14890            interrupt.park_because("higher-priority task waiting");
14891            // Nothing the node does can observe this yet - there is no
14892            // check inside it, by construction - so let the executor run
14893            // anything pending and then let the node finish on its own.
14894            tokio::task::yield_now().await;
14895            finish_tx.send(()).expect("send finish");
14896        };
14897
14898        let (node_result, ()) = tokio::join!(node, interrupter);
14899        assert_eq!(
14900            node_result, "node finished",
14901            "the in-flight call ran to completion"
14902        );
14903
14904        // Only now, at the boundary the real `execute` would check right
14905        // after this node, does the park take effect.
14906        assert!(runner.park_here().expect("park_here"));
14907        assert!(runner.state.parked);
14908    }
14909
14910    /// A run parked mid-competition carries every field it had accumulated
14911    /// through the exact same disk round-trip an ordinary resume uses -
14912    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
14913    /// Nothing about parking for an interrupt is a special case of that path;
14914    /// this is what proves it rather than assuming it.
14915    #[test]
14916    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14917        crate::run::pin_test_home();
14918        let mut runner = runner_at(RunStatus::Judging);
14919        // `Runner::resume` re-resolves roles from the saved config, which
14920        // refuses an empty roster - give it the same minimal one `conductor`
14921        // itself uses.
14922        runner.state.config.agents = vec![conductor()];
14923        runner.state.candidates = vec![Candidate {
14924            index: 0,
14925            label: 'A',
14926            agent: "alpha".to_owned(),
14927            branch: "magi/x/A".to_owned(),
14928            worktree: PathBuf::from("/nonexistent/worktree"),
14929            summary: "did the thing".to_owned(),
14930            stat: "1 file changed".to_owned(),
14931            files: 1,
14932            commits: 1,
14933            empty: false,
14934            failed: None,
14935            verified_noop: None,
14936            duration_ms: 1234,
14937            folded: false,
14938        }];
14939        let run_id = runner.state.id.clone();
14940
14941        let interrupt = Pause::new();
14942        runner.watch_interrupt(interrupt.clone());
14943        interrupt.park_because("task c3d4 asked to run first");
14944        assert!(runner.park_here().expect("park_here"));
14945
14946        let resumed = Runner::resume(&run_id).expect("resume");
14947        assert_eq!(resumed.state.candidates.len(), 1);
14948        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14949        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14950        assert_eq!(resumed.state.status, runner.state.status);
14951        assert!(
14952            resumed.state.parked,
14953            "still parked until `execute` actually walks the graph again"
14954        );
14955        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14956    }
14957
14958    /// A fresh open question on `run`, stored and handed back for assertions.
14959    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14960        let mut q = ask::Question::new(
14961            run.to_owned(),
14962            "implement".to_owned(),
14963            "impl-A".to_owned(),
14964            "Which storage backend should the cache use?".to_owned(),
14965            String::new(),
14966            vec!["SQLite".to_owned(), "Redis".to_owned()],
14967        );
14968        store.put(&mut q).unwrap();
14969        q
14970    }
14971
14972    #[test]
14973    fn a_failed_runs_open_question_is_abandoned() {
14974        ask_test_home();
14975        let store = ask::Questions::open();
14976        let mut runner = runner_at(RunStatus::Failed);
14977        let run = runner.state.id.clone();
14978        let q = ask_open_question(&store, &run);
14979
14980        runner.settle_questions();
14981
14982        let back = store.get(&q.id).unwrap();
14983        assert!(
14984            !back.status.open(),
14985            "the seat that asked died with the run; nobody is left to read an answer"
14986        );
14987        assert!(
14988            back.detail.contains(&run) && back.detail.contains("failed"),
14989            "the reason names what the run became, not just that it is gone: {}",
14990            back.detail
14991        );
14992    }
14993
14994    #[test]
14995    fn a_merged_runs_open_question_is_abandoned_too() {
14996        ask_test_home();
14997        let store = ask::Questions::open();
14998        // A run that finishes cleanly still leaves nobody to read an answer -
14999        // this is not only a failure-path cleanup.
15000        for status in [RunStatus::Merged, RunStatus::Ready] {
15001            let mut runner = runner_at(status);
15002            let run = runner.state.id.clone();
15003            let q = ask_open_question(&store, &run);
15004
15005            runner.settle_questions();
15006
15007            let back = store.get(&q.id).unwrap();
15008            assert!(
15009                !back.status.open(),
15010                "{status:?} run's question must not outlive the run"
15011            );
15012        }
15013    }
15014
15015    #[test]
15016    fn a_still_resumable_runs_open_question_is_left_alone() {
15017        ask_test_home();
15018        let store = ask::Questions::open();
15019        // `Blocked` and `Stalled` can still be resumed — the candidates, the
15020        // review round and the seat sessions are all still on disk — so a
15021        // question asked mid-round may yet get a real answer from a real
15022        // resume. Sweeping it here would be exactly the failure mode this
15023        // whole feature exists to avoid on the other side.
15024        for status in [RunStatus::Blocked, RunStatus::Stalled] {
15025            let mut runner = runner_at(status);
15026            let run = runner.state.id.clone();
15027            let q = ask_open_question(&store, &run);
15028
15029            runner.settle_questions();
15030
15031            let back = store.get(&q.id).unwrap();
15032            assert!(
15033                back.status.open(),
15034                "{status:?} is still alive; the question must still be waiting"
15035            );
15036        }
15037    }
15038
15039    #[test]
15040    fn settle_questions_never_touches_an_already_answered_question() {
15041        ask_test_home();
15042        let store = ask::Questions::open();
15043        let mut runner = runner_at(RunStatus::Failed);
15044        let run = runner.state.id.clone();
15045        let mut q = ask_open_question(&store, &run);
15046        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15047            .unwrap();
15048        store.put(&mut q).unwrap();
15049
15050        // Called twice, the way a crash-recovered daemon reclaim and the
15051        // graph's own cleanup both can for the same run — `abandon_for_run`
15052        // only ever touches what is still open, so this must be inert both
15053        // times, not merely the second.
15054        runner.settle_questions();
15055        runner.settle_questions();
15056
15057        let back = store.get(&q.id).unwrap();
15058        assert_eq!(
15059            back.status,
15060            ask::QuestionStatus::Answered,
15061            "a real answer is a decision on record, never overwritten by a sweep"
15062        );
15063    }
15064
15065    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15066    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15067    /// without merging, whose winner is still the operator's answer to read.
15068    /// Nothing previously called `fold_run` itself with a real `tally`, so
15069    /// this is the first test to pin down the one distinction the whole
15070    /// automatic-fold feature depends on: the winner's worktree and branch
15071    /// must survive, everything else sharing the run's worktree bay - a
15072    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15073    /// second sweep treats every non-winner directory under the bay alike -
15074    /// must not.
15075    #[tokio::test]
15076    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15077        crate::run::pin_test_home();
15078        let tmp = tempfile::tempdir().expect("tempdir");
15079        let repo = tmp.path().join("repo");
15080        std::fs::create_dir_all(&repo).unwrap();
15081        init_repo(&repo);
15082
15083        let mut config = Config::default();
15084        config.graph.worktree_root = Some(tmp.path().join("wt"));
15085
15086        let mut state = RunState::new(
15087            repo.clone(),
15088            "main".to_owned(),
15089            "deadbeef".to_owned(),
15090            "task".to_owned(),
15091            config,
15092        );
15093        let root = state.worktree_root();
15094        let wt_a = root.join("cand-A");
15095        let wt_b = root.join("cand-B");
15096        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15097            .await
15098            .expect("worktree A");
15099        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15100            .await
15101            .expect("worktree B");
15102
15103        state.candidates = vec![
15104            Candidate {
15105                index: 0,
15106                label: 'A',
15107                agent: "alpha".to_owned(),
15108                branch: "magi/x/A".to_owned(),
15109                worktree: wt_a.clone(),
15110                summary: String::new(),
15111                stat: String::new(),
15112                files: 0,
15113                commits: 0,
15114                empty: false,
15115                failed: None,
15116                verified_noop: None,
15117                duration_ms: 0,
15118                folded: false,
15119            },
15120            Candidate {
15121                index: 1,
15122                label: 'B',
15123                agent: "beta".to_owned(),
15124                branch: "magi/x/B".to_owned(),
15125                worktree: wt_b.clone(),
15126                summary: String::new(),
15127                stat: String::new(),
15128                files: 0,
15129                commits: 0,
15130                empty: false,
15131                failed: None,
15132                verified_noop: None,
15133                duration_ms: 0,
15134                folded: false,
15135            },
15136        ];
15137        state.tally = Some(Tally {
15138            first_choice: BTreeMap::from([('A', 1)]),
15139            borda: BTreeMap::new(),
15140            winner: 'A',
15141            rankings: 1,
15142            unanimous_initial: true,
15143            deliberated: false,
15144            changed_votes: 0,
15145            unanimous_final: true,
15146            tie_break: None,
15147            judges: 1,
15148            present: 1,
15149            quorum: 1,
15150            met_quorum: true,
15151            uncontested: None,
15152        });
15153        state.status = RunStatus::Ready;
15154
15155        fold_run(&mut state, false, &crate::run::home())
15156            .await
15157            .expect("fold_run");
15158
15159        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15160        assert!(
15161            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15162            "the unmerged winner's branch survives"
15163        );
15164        assert!(
15165            !state.candidates[0].folded,
15166            "the winner is not marked folded"
15167        );
15168
15169        assert!(!wt_b.exists(), "the loser's worktree is removed");
15170        assert!(
15171            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15172            "the loser's branch is removed"
15173        );
15174        assert!(state.candidates[1].folded, "the loser is marked folded");
15175    }
15176
15177    #[tokio::test]
15178    #[cfg(unix)]
15179    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15180        use std::os::fd::AsRawFd as _;
15181        use std::time::Duration;
15182
15183        let tmp = tempfile::tempdir().unwrap();
15184        let repo = tmp.path().join("repo");
15185        std::fs::create_dir_all(&repo).unwrap();
15186        init_repo(&repo);
15187        let common = repo.join(".git");
15188        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15189        let home = tmp.path().join("home");
15190        let mut config = Config::default();
15191        config.coordination = crate::config::test_coordination();
15192        config.graph.worktree_root = Some(tmp.path().join("wt"));
15193        config.graph.timeout_resource_wait = 1;
15194        let mut state = RunState::new(
15195            repo.clone(),
15196            "main".into(),
15197            "deadbeef".into(),
15198            "task".into(),
15199            config,
15200        );
15201        let root = state.worktree_root();
15202        let candidate = root.join("cand-A");
15203        let reviewer = root.join("review-1");
15204        let branch = state.branch_for('A');
15205        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15206            .await
15207            .unwrap();
15208        git::worktree_add_detached(&repo, &reviewer, "main")
15209            .await
15210            .unwrap();
15211        git::acquire_worktree_config(&repo).await.unwrap();
15212        state.enabled_worktree_config = true;
15213        state.status = RunStatus::Ready;
15214        state.candidates.push(Candidate {
15215            index: 0,
15216            label: 'A',
15217            agent: "implement".into(),
15218            branch: branch.clone(),
15219            worktree: candidate.clone(),
15220            summary: String::new(),
15221            stat: String::new(),
15222            files: 0,
15223            commits: 0,
15224            empty: false,
15225            failed: None,
15226            verified_noop: None,
15227            duration_ms: 0,
15228            folded: false,
15229        });
15230        state.save_under(&home).unwrap();
15231        let config_before = std::fs::read(common.join("config")).unwrap();
15232        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15233            .await
15234            .unwrap();
15235        // A separate OS handle bypasses the launcher's local gate, exactly as
15236        // an external updater does, and must still prevent all folding work.
15237        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15238        assert_eq!(
15239            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15240            0
15241        );
15242        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15243        assert!(
15244            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15245                .await
15246                .is_err()
15247        );
15248        assert!(candidate.exists());
15249        assert!(reviewer.exists());
15250        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15251        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15252        assert_eq!(
15253            git::git(&repo, &["worktree", "list", "--porcelain"])
15254                .await
15255                .unwrap(),
15256            worktrees_before
15257        );
15258        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15259            .await
15260            .unwrap()
15261            .unwrap_err();
15262        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15263        drop(fold);
15264        assert!(candidate.exists() && reviewer.exists());
15265        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15266        assert_eq!(
15267            state.resource_block.as_ref().unwrap().kind,
15268            "repository coordination"
15269        );
15270        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15271        drop(external);
15272        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15273        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15274            .await
15275            .unwrap()
15276            .unwrap();
15277        drop(fold);
15278        assert!(removed.contains(&branch));
15279        assert!(!root.exists());
15280        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15281        assert!(state.candidates[0].folded);
15282        assert!(!state.enabled_worktree_config);
15283        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15284        let enabled = git::git_raw(
15285            &repo,
15286            &["config", "--local", "--get", "extensions.worktreeConfig"],
15287        )
15288        .await
15289        .unwrap();
15290        assert!(!enabled.ok(), "the last config reference was released");
15291    }
15292
15293    /// A branch handed to a later run is that run's (and its pull request's):
15294    /// folding the run that released it must not delete it.
15295    #[tokio::test]
15296    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15297        let tmp = tempfile::tempdir().expect("tempdir");
15298        let repo = tmp.path().join("repo");
15299        std::fs::create_dir_all(&repo).unwrap();
15300        init_repo(&repo);
15301        let home = tmp.path().join("home");
15302
15303        let mut config = Config::default();
15304        config.graph.worktree_root = Some(tmp.path().join("wt"));
15305        let mut state = RunState::new(
15306            repo.clone(),
15307            "main".to_owned(),
15308            "deadbeef".to_owned(),
15309            "task".to_owned(),
15310            config,
15311        );
15312        // The worktree is already gone (released); the branch survives.
15313        git::git(&repo, &["branch", "magi/x/A", "main"])
15314            .await
15315            .expect("branch");
15316        state.candidates = vec![Candidate {
15317            index: 0,
15318            label: 'A',
15319            agent: "alpha".to_owned(),
15320            branch: "magi/x/A".to_owned(),
15321            worktree: state.worktree_root().join("cand-A"),
15322            summary: String::new(),
15323            stat: String::new(),
15324            files: 0,
15325            commits: 0,
15326            empty: false,
15327            failed: None,
15328            verified_noop: None,
15329            duration_ms: 0,
15330            folded: true,
15331        }];
15332        state.released_to = Some("20260901-000000-new1".to_owned());
15333        state.released_branches = vec!["magi/x/A".to_owned()];
15334
15335        fold_run(&mut state, true, &home).await.expect("fold_run");
15336
15337        assert!(
15338            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15339            "the handed-over branch survives a fold"
15340        );
15341    }
15342
15343    /// A winner with nothing ahead of the base is caught before `gh` is ever
15344    /// asked for a pull request, and the message carries what the task's
15345    /// references resolved to.
15346    #[tokio::test]
15347    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15348        let tmp = tempfile::tempdir().expect("tempdir");
15349        let repo = tmp.path().join("repo");
15350        std::fs::create_dir_all(&repo).unwrap();
15351        init_repo(&repo);
15352        let run = |args: &[&str]| {
15353            let out = std::process::Command::new("git")
15354                .quiet()
15355                .args(args)
15356                .current_dir(&repo)
15357                .output()
15358                .expect("spawn git");
15359            assert!(out.status.success(), "git {args:?}");
15360        };
15361        run(&["branch", "magi/x/A"]);
15362        run(&["checkout", "-q", "-b", "magi/x/B"]);
15363        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15364        run(&["add", "-A"]);
15365        run(&["commit", "-q", "-m", "work"]);
15366        run(&["checkout", "-q", "main"]);
15367
15368        // A pull request is compared against the remote's base, so the
15369        // fixture needs one. Before it exists nothing can be read, and the
15370        // answer must be "not empty".
15371        let probe = RunState::new(
15372            repo.clone(),
15373            "main".to_owned(),
15374            "deadbeef".to_owned(),
15375            "task".to_owned(),
15376            Config::default(),
15377        );
15378        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15379        let bare = tmp.path().join("origin.git");
15380        let out = std::process::Command::new("git")
15381            .quiet()
15382            .args(["init", "-q", "--bare"])
15383            .arg(&bare)
15384            .output()
15385            .expect("spawn git");
15386        assert!(out.status.success(), "git init --bare");
15387        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15388        run(&["push", "-q", "origin", "main"]);
15389
15390        let mut state = RunState::new(
15391            repo.clone(),
15392            "main".to_owned(),
15393            "deadbeef".to_owned(),
15394            "task".to_owned(),
15395            Config::default(),
15396        );
15397        state.seeds = vec![refs::Seed {
15398            token: "magi/27b2/A".to_owned(),
15399            kind: refs::SeedKind::Unresolved,
15400            sha: String::new(),
15401            branch: true,
15402            detail: "no branch or commit named magi/27b2/A".to_owned(),
15403        }];
15404
15405        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15406        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15407        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15408        let detail = empty_candidate_detail(&state, "main");
15409        assert!(detail.starts_with("empty candidate"), "{detail}");
15410        assert!(!detail.contains("magi/27b2/A"), "{detail}");
15411        assert!(detail.contains("--start-from"), "{detail}");
15412    }
15413
15414    /// `status == Ready` used to be read as "this is the harmless
15415    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15416    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15417    /// run's PR was closed without merging — and reentering `merge` with
15418    /// `mode` still `Pr` does not know the difference, so it pushed and
15419    /// opened a second pull request. `mode == Local` reproduces the same
15420    /// blind spot without a network call: reentry must not attempt another
15421    /// git merge once this node has already recorded an outcome.
15422    #[tokio::test]
15423    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15424        let tmp = tempfile::tempdir().expect("tempdir");
15425        let repo = tmp.path().join("repo");
15426        std::fs::create_dir_all(&repo).unwrap();
15427        init_repo(&repo);
15428
15429        let mut config = Config::default();
15430        config.merge.mode = MergeMode::Local;
15431
15432        let mut state = RunState::new(
15433            repo.clone(),
15434            "main".to_owned(),
15435            "deadbeef".to_owned(),
15436            "task".to_owned(),
15437            config,
15438        );
15439        state.candidates = vec![Candidate {
15440            index: 0,
15441            label: 'A',
15442            agent: "alpha".to_owned(),
15443            branch: "does-not-exist".to_owned(),
15444            worktree: repo.clone(),
15445            summary: String::new(),
15446            stat: String::new(),
15447            files: 0,
15448            commits: 0,
15449            empty: false,
15450            failed: None,
15451            verified_noop: None,
15452            duration_ms: 0,
15453            folded: false,
15454        }];
15455        state.tally = Some(Tally {
15456            first_choice: BTreeMap::from([('A', 1)]),
15457            borda: BTreeMap::new(),
15458            winner: 'A',
15459            rankings: 1,
15460            unanimous_initial: true,
15461            deliberated: false,
15462            changed_votes: 0,
15463            unanimous_final: true,
15464            tie_break: None,
15465            judges: 0,
15466            present: 0,
15467            quorum: 0,
15468            met_quorum: true,
15469            uncontested: Some("only candidate A produced a change".to_owned()),
15470        });
15471        state.reviews = vec![ReviewRound {
15472            round: 1,
15473            head: "deadbeef".to_owned(),
15474            verified_head: None,
15475            verified_at: None,
15476            reviews: Vec::new(),
15477            e2e: Vec::new(),
15478            fix: None,
15479            blocking: 0,
15480            answered: 0,
15481            expected: 0,
15482            clean: true,
15483            verify_retried: false,
15484            e2e_deferred: false,
15485            e2e_defer_reason: None,
15486            progressed: false,
15487            vote_split: false,
15488            reconsideration: Vec::new(),
15489            verdict: None,
15490        }];
15491        state.gate = vec![CommandOutcome {
15492            command: "test".to_owned(),
15493            code: Some(0),
15494            output_tail: String::new(),
15495            duration_ms: 0,
15496            resource_blocked: false,
15497        }];
15498        state.gate_ran = true;
15499        // Reached its conclusion already — e.g. `land` closing the PR without
15500        // merging it, which (like the honest `MergeMode::None` path) leaves
15501        // `status` at `Ready`. The recorded outcome is what actually marks
15502        // this node done.
15503        state.status = RunStatus::Ready;
15504        state.merge = Some(MergeOutcome {
15505            mode: MergeMode::Local,
15506            ok: false,
15507            detail: "already concluded".to_owned(),
15508            empty: false,
15509        });
15510
15511        let mut runner = Runner {
15512            state,
15513            roles: ResolvedRoles {
15514                implementers: Vec::new(),
15515                judges: Vec::new(),
15516                reviewers: Vec::new(),
15517                fixer: None,
15518                conductor: conductor(),
15519                implementer_roster: Vec::new(),
15520                judge_roster: Vec::new(),
15521                reviewer_roster: Vec::new(),
15522            },
15523            sem: Arc::new(Semaphore::new(1)),
15524            pause: Pause::new(),
15525            interrupt: Pause::new(),
15526        };
15527
15528        runner.merge().await.expect("merge");
15529
15530        assert_eq!(
15531            runner.state.status,
15532            RunStatus::Ready,
15533            "a concluded run's status must not change on reentry"
15534        );
15535        assert_eq!(
15536            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15537            Some("already concluded"),
15538            "merge must not run again once the node already recorded an outcome"
15539        );
15540    }
15541
15542    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15543    /// when its last attempt was resource-blocked (the shared build cache
15544    /// could not be acquired or confirmed fresh in time - see
15545    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15546    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15547    /// a run merge a tree the gate never actually checked - exactly the case
15548    /// a contended cache produces on every retry until it clears. `merge`
15549    /// must refuse until `gate` has actually recorded an attempt.
15550    #[tokio::test]
15551    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15552        let tmp = tempfile::tempdir().expect("tempdir");
15553        let repo = tmp.path().join("repo");
15554        std::fs::create_dir_all(&repo).unwrap();
15555        init_repo(&repo);
15556
15557        let mut config = Config::default();
15558        config.merge.mode = MergeMode::Local;
15559
15560        let mut state = RunState::new(
15561            repo.clone(),
15562            "main".to_owned(),
15563            "deadbeef".to_owned(),
15564            "task".to_owned(),
15565            config,
15566        );
15567        state.candidates = vec![Candidate {
15568            index: 0,
15569            label: 'A',
15570            agent: "alpha".to_owned(),
15571            branch: "does-not-exist".to_owned(),
15572            worktree: repo.clone(),
15573            summary: String::new(),
15574            stat: String::new(),
15575            files: 0,
15576            commits: 0,
15577            empty: false,
15578            failed: None,
15579            verified_noop: None,
15580            duration_ms: 0,
15581            folded: false,
15582        }];
15583        state.tally = Some(Tally {
15584            first_choice: BTreeMap::from([('A', 1)]),
15585            borda: BTreeMap::new(),
15586            winner: 'A',
15587            rankings: 1,
15588            unanimous_initial: true,
15589            deliberated: false,
15590            changed_votes: 0,
15591            unanimous_final: true,
15592            tie_break: None,
15593            judges: 0,
15594            present: 0,
15595            quorum: 0,
15596            met_quorum: true,
15597            uncontested: Some("only candidate A produced a change".to_owned()),
15598        });
15599        state.reviews = vec![ReviewRound {
15600            round: 1,
15601            head: "deadbeef".to_owned(),
15602            verified_head: None,
15603            verified_at: None,
15604            reviews: Vec::new(),
15605            e2e: Vec::new(),
15606            fix: None,
15607            blocking: 0,
15608            answered: 0,
15609            expected: 0,
15610            clean: true,
15611            verify_retried: false,
15612            e2e_deferred: false,
15613            e2e_defer_reason: None,
15614            progressed: false,
15615            vote_split: false,
15616            reconsideration: Vec::new(),
15617            verdict: None,
15618        }];
15619        // The point: `gate` has not recorded anything yet.
15620        state.gate = Vec::new();
15621        state.gate_ran = false;
15622        state.status = RunStatus::Gating;
15623
15624        let mut runner = Runner {
15625            state,
15626            roles: ResolvedRoles {
15627                implementers: Vec::new(),
15628                judges: Vec::new(),
15629                reviewers: Vec::new(),
15630                fixer: None,
15631                conductor: conductor(),
15632                implementer_roster: Vec::new(),
15633                judge_roster: Vec::new(),
15634                reviewer_roster: Vec::new(),
15635            },
15636            sem: Arc::new(Semaphore::new(1)),
15637            pause: Pause::new(),
15638            interrupt: Pause::new(),
15639        };
15640
15641        runner.merge().await.expect("merge");
15642
15643        assert!(
15644            runner.state.merge.is_none(),
15645            "an empty gate must never be read as a passing one: {:?}",
15646            runner.state.merge
15647        );
15648    }
15649
15650    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
15651    /// commands configured and `merge.mode` is `none` (a review-only run).
15652    /// `gate` must still record a real attempt — zero commands, vacuously
15653    /// passed — rather than leaving `state.gate` empty in a way `merge`
15654    /// cannot tell apart from "never ran"; otherwise the run reaches
15655    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
15656    #[tokio::test]
15657    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15658        let tmp = tempfile::tempdir().expect("tempdir");
15659        let repo = tmp.path().join("repo");
15660        std::fs::create_dir_all(&repo).unwrap();
15661        init_repo(&repo);
15662
15663        // Default config: `verify.gate` empty, `merge.mode` is `none`.
15664        let config = Config::default();
15665
15666        let mut state = RunState::new(
15667            repo.clone(),
15668            "main".to_owned(),
15669            "deadbeef".to_owned(),
15670            "task".to_owned(),
15671            config,
15672        );
15673        state.candidates = vec![Candidate {
15674            index: 0,
15675            label: 'A',
15676            agent: "alpha".to_owned(),
15677            branch: "does-not-exist".to_owned(),
15678            worktree: repo.clone(),
15679            summary: String::new(),
15680            stat: String::new(),
15681            files: 0,
15682            commits: 0,
15683            empty: false,
15684            failed: None,
15685            verified_noop: None,
15686            duration_ms: 0,
15687            folded: false,
15688        }];
15689        state.tally = Some(Tally {
15690            first_choice: BTreeMap::from([('A', 1)]),
15691            borda: BTreeMap::new(),
15692            winner: 'A',
15693            rankings: 1,
15694            unanimous_initial: true,
15695            deliberated: false,
15696            changed_votes: 0,
15697            unanimous_final: true,
15698            tie_break: None,
15699            judges: 0,
15700            present: 0,
15701            quorum: 0,
15702            met_quorum: true,
15703            uncontested: Some("only candidate A produced a change".to_owned()),
15704        });
15705        state.reviews = vec![ReviewRound {
15706            round: 1,
15707            head: "deadbeef".to_owned(),
15708            verified_head: None,
15709            verified_at: None,
15710            reviews: Vec::new(),
15711            e2e: Vec::new(),
15712            fix: None,
15713            blocking: 0,
15714            answered: 0,
15715            expected: 0,
15716            clean: true,
15717            verify_retried: false,
15718            e2e_deferred: false,
15719            e2e_defer_reason: None,
15720            progressed: false,
15721            vote_split: false,
15722            reconsideration: Vec::new(),
15723            verdict: None,
15724        }];
15725
15726        let mut runner = Runner {
15727            state,
15728            roles: ResolvedRoles {
15729                implementers: Vec::new(),
15730                judges: Vec::new(),
15731                reviewers: Vec::new(),
15732                fixer: None,
15733                conductor: conductor(),
15734                implementer_roster: Vec::new(),
15735                judge_roster: Vec::new(),
15736                reviewer_roster: Vec::new(),
15737            },
15738            sem: Arc::new(Semaphore::new(1)),
15739            pause: Pause::new(),
15740            interrupt: Pause::new(),
15741        };
15742
15743        runner.gate().await.expect("gate");
15744        assert!(
15745            runner.state.gate_ran,
15746            "zero configured commands is still a real attempt, not an unrun gate"
15747        );
15748        assert!(runner.state.gate.is_empty());
15749        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15750        assert_ne!(
15751            runner.state.status,
15752            RunStatus::Blocked,
15753            "a gate with nothing to check must not read as failed"
15754        );
15755
15756        runner.merge().await.expect("merge");
15757        assert_eq!(
15758            runner.state.status,
15759            RunStatus::Ready,
15760            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15761        );
15762    }
15763
15764    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15765    /// `verify.gate` (so the e2e leg and the final gate never build against
15766    /// different directories). With zero `verify.gate` commands but a
15767    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15768    /// that lease before discovering it had nothing to run - so a repo with
15769    /// no gate commands could come back `resource_blocked` (and therefore
15770    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15771    /// cache, exactly the contention this run's own zero commands could
15772    /// never have touched. `gate` must recognise there is nothing to check
15773    /// before it ever asks for the lease.
15774    #[tokio::test]
15775    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15776        crate::run::pin_test_home();
15777        let home = crate::run::home();
15778
15779        let tmp = tempfile::tempdir().expect("tempdir");
15780        let repo = tmp.path().join("repo");
15781        std::fs::create_dir_all(&repo).unwrap();
15782        init_repo(&repo);
15783        // Unique to this test, so holding its lease cannot collide with
15784        // another test sharing the same process-wide `home`.
15785        let cache_dir = tmp.path().join("target");
15786
15787        let mut config = Config::default();
15788        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15789        // `verify.gate` stays empty (the default). Bounded so a regression
15790        // that does start waiting fails the test in seconds, not hangs it.
15791        config.graph.timeout_verify = Some(2);
15792        config.graph.timeout_resource_wait = 2;
15793
15794        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15795        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15796            .expect("no io error acquiring directly")
15797        {
15798            crate::cache::AcquireOutcome::Acquired(g) => g,
15799            crate::cache::AcquireOutcome::Busy(b) => {
15800                panic!("expected the direct acquire to win the lease first: {b:?}")
15801            }
15802        };
15803
15804        let mut state = RunState::new(
15805            repo.clone(),
15806            "main".to_owned(),
15807            "deadbeef".to_owned(),
15808            "task".to_owned(),
15809            config,
15810        );
15811        state.candidates = vec![Candidate {
15812            index: 0,
15813            label: 'A',
15814            agent: "alpha".to_owned(),
15815            branch: "does-not-exist".to_owned(),
15816            worktree: repo.clone(),
15817            summary: String::new(),
15818            stat: String::new(),
15819            files: 0,
15820            commits: 0,
15821            empty: false,
15822            failed: None,
15823            verified_noop: None,
15824            duration_ms: 0,
15825            folded: false,
15826        }];
15827        state.tally = Some(Tally {
15828            first_choice: BTreeMap::from([('A', 1)]),
15829            borda: BTreeMap::new(),
15830            winner: 'A',
15831            rankings: 1,
15832            unanimous_initial: true,
15833            deliberated: false,
15834            changed_votes: 0,
15835            unanimous_final: true,
15836            tie_break: None,
15837            judges: 0,
15838            present: 0,
15839            quorum: 0,
15840            met_quorum: true,
15841            uncontested: Some("only candidate A produced a change".to_owned()),
15842        });
15843        state.reviews = vec![ReviewRound {
15844            round: 1,
15845            head: "deadbeef".to_owned(),
15846            verified_head: None,
15847            verified_at: None,
15848            reviews: Vec::new(),
15849            e2e: Vec::new(),
15850            fix: None,
15851            blocking: 0,
15852            answered: 0,
15853            expected: 0,
15854            clean: true,
15855            verify_retried: false,
15856            e2e_deferred: false,
15857            e2e_defer_reason: None,
15858            progressed: false,
15859            vote_split: false,
15860            reconsideration: Vec::new(),
15861            verdict: None,
15862        }];
15863
15864        let mut runner = Runner {
15865            state,
15866            roles: ResolvedRoles {
15867                implementers: Vec::new(),
15868                judges: Vec::new(),
15869                reviewers: Vec::new(),
15870                fixer: None,
15871                conductor: conductor(),
15872                implementer_roster: Vec::new(),
15873                judge_roster: Vec::new(),
15874                reviewer_roster: Vec::new(),
15875            },
15876            sem: Arc::new(Semaphore::new(1)),
15877            pause: Pause::new(),
15878            interrupt: Pause::new(),
15879        };
15880
15881        let started = std::time::Instant::now();
15882        runner.gate().await.expect("gate");
15883        assert!(
15884            started.elapsed() < Duration::from_secs(1),
15885            "a gate with nothing to run must never wait on a lease it never needed"
15886        );
15887        assert!(
15888            runner.state.gate_ran,
15889            "zero commands is still a real, immediate attempt"
15890        );
15891        assert!(runner.state.gate.is_empty());
15892        assert_ne!(
15893            runner.state.status,
15894            RunStatus::Blocked,
15895            "must not read as resource-blocked on a lease it never asked for"
15896        );
15897    }
15898
15899    #[cfg(unix)]
15900    #[tokio::test]
15901    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15902        crate::run::pin_test_home();
15903        let tmp = tempfile::tempdir().unwrap();
15904        let mut config = Config::default();
15905        config.disk.min_free_bytes = 0;
15906        let mut state = RunState::new(
15907            tmp.path().to_owned(),
15908            "main".into(),
15909            "base".into(),
15910            "verification evidence".into(),
15911            config,
15912        );
15913        let (outcomes, pids) = run_commands(
15914            &mut state,
15915            "gate",
15916            "gate",
15917            1,
15918            &["sh".into(), "-c".into()],
15919            &[
15920                "printf before-timeout; exec sleep 30".into(),
15921                "kill -TERM $$".into(),
15922                "exit 7".into(),
15923                "printf success".into(),
15924            ],
15925            tmp.path(),
15926            Duration::from_secs(1),
15927        )
15928        .await;
15929        wait_for_timed_out_children_to_die(&pids).await;
15930        assert_eq!(
15931            pids.len(),
15932            1,
15933            "unleased timeout retains its death-check PID"
15934        );
15935        assert_eq!(outcomes.len(), 4);
15936        assert_eq!(outcomes[0].code, None);
15937        assert!(outcomes[0].output_tail.contains("before-timeout"));
15938        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15939        assert_eq!(outcomes[1].code, None);
15940        assert!(outcomes[1].output_tail.contains("command terminated:"));
15941        assert!(!outcomes[1].output_tail.contains("timed out"));
15942        assert_eq!(outcomes[2].code, Some(7));
15943        assert!(!outcomes[2].output_tail.contains("timed out"));
15944        assert_eq!(outcomes[3].code, Some(0));
15945        assert_eq!(outcomes[3].output_tail, "success");
15946        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15947    }
15948
15949    /// The addendum's second gap: a `verify.gate` command running for real
15950    /// wall-clock time had nothing at all to show for it in `active` before
15951    /// `run_commands` learned to record it — a run could sit in `Gating` for
15952    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
15953    /// what was actually happening. Proven with a genuinely still-running
15954    /// command, not just a before/after check on the final state: a poller
15955    /// task reads the same `run.json` `gate()` is writing, the same way the
15956    /// phone or `magi show` would, while the shell command is still blocked
15957    /// on its own release marker.
15958    #[tokio::test]
15959    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15960        crate::run::pin_test_home();
15961
15962        let tmp = tempfile::tempdir().expect("tempdir");
15963        let repo = tmp.path().join("repo");
15964        std::fs::create_dir_all(&repo).unwrap();
15965        init_repo(&repo);
15966
15967        let mut config = Config::default();
15968        config.verify.gate = vec![
15969            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15970             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15971                .to_owned(),
15972        ];
15973
15974        let mut state = RunState::new(
15975            repo.clone(),
15976            "main".to_owned(),
15977            "deadbeef".to_owned(),
15978            "task".to_owned(),
15979            config,
15980        );
15981        let run_id = state.id.clone();
15982        state.candidates = vec![Candidate {
15983            index: 0,
15984            label: 'A',
15985            agent: "alpha".to_owned(),
15986            branch: "does-not-exist".to_owned(),
15987            worktree: repo.clone(),
15988            summary: String::new(),
15989            stat: String::new(),
15990            files: 0,
15991            commits: 0,
15992            empty: false,
15993            failed: None,
15994            verified_noop: None,
15995            duration_ms: 0,
15996            folded: false,
15997        }];
15998        state.tally = Some(Tally {
15999            first_choice: BTreeMap::from([('A', 1)]),
16000            borda: BTreeMap::new(),
16001            winner: 'A',
16002            rankings: 1,
16003            unanimous_initial: true,
16004            deliberated: false,
16005            changed_votes: 0,
16006            unanimous_final: true,
16007            tie_break: None,
16008            judges: 0,
16009            present: 0,
16010            quorum: 0,
16011            met_quorum: true,
16012            uncontested: Some("only candidate A produced a change".to_owned()),
16013        });
16014        state.reviews = vec![ReviewRound {
16015            round: 1,
16016            head: "deadbeef".to_owned(),
16017            verified_head: None,
16018            verified_at: None,
16019            reviews: Vec::new(),
16020            e2e: Vec::new(),
16021            fix: None,
16022            blocking: 0,
16023            answered: 0,
16024            expected: 0,
16025            clean: true,
16026            verify_retried: false,
16027            e2e_deferred: false,
16028            e2e_defer_reason: None,
16029            progressed: false,
16030            vote_split: false,
16031            reconsideration: Vec::new(),
16032            verdict: None,
16033        }];
16034
16035        let mut runner = Runner {
16036            state,
16037            roles: ResolvedRoles {
16038                implementers: Vec::new(),
16039                judges: Vec::new(),
16040                reviewers: Vec::new(),
16041                fixer: None,
16042                conductor: conductor(),
16043                implementer_roster: Vec::new(),
16044                judge_roster: Vec::new(),
16045                reviewer_roster: Vec::new(),
16046            },
16047            sem: Arc::new(Semaphore::new(1)),
16048            pause: Pause::new(),
16049            interrupt: Pause::new(),
16050        };
16051
16052        let started_marker = repo.join("started.marker");
16053        let release_marker = repo.join("release.marker");
16054        let poller = tokio::spawn(async move {
16055            // Bounded so a regression that never records the task entry
16056            // fails this test in seconds instead of hanging the suite —
16057            // the same shape `a_park_requested_while_a_seat_is_mid_call_
16058            // does_not_cut_it_short` uses for the same reason.
16059            for _ in 0..100 {
16060                if started_marker.exists()
16061                    && let Ok(s) = crate::run::RunState::load(&run_id)
16062                    && let Some(a) = s.active.get("gate")
16063                {
16064                    std::fs::write(&release_marker, b"go").expect("release marker");
16065                    return Some(a.clone());
16066                }
16067                tokio::time::sleep(Duration::from_millis(50)).await;
16068            }
16069            None
16070        });
16071
16072        runner.gate().await.expect("gate");
16073        let captured = poller.await.expect("poller task");
16074        let captured = captured.expect(
16075            "the poller never saw a `gate` task entry in run.json while the command was \
16076             still blocked on its own release marker",
16077        );
16078
16079        assert_eq!(captured.task.as_deref(), Some("gate"));
16080        assert_eq!(captured.node, "gate");
16081        assert_eq!(captured.index, Some(1));
16082        assert_eq!(captured.total, Some(1));
16083        assert!(
16084            captured
16085                .command
16086                .as_deref()
16087                .is_some_and(|c| c.contains("started.marker")),
16088            "{captured:?}"
16089        );
16090
16091        assert!(
16092            runner.state.active.is_empty(),
16093            "the entry must be cleared once the command actually finished: {:?}",
16094            runner.state.active
16095        );
16096        assert!(runner.state.gate_ran);
16097        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16098    }
16099
16100    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16101    /// record for `land`; one with only a Minor, or no reject, leaves none.
16102    #[tokio::test]
16103    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16104        use crate::verdict::{Finding, ReviewVote, Severity};
16105        crate::run::pin_test_home();
16106        let tmp = tempfile::tempdir().expect("tempdir");
16107        let repo = tmp.path().join("repo");
16108        std::fs::create_dir_all(&repo).unwrap();
16109        init_repo(&repo);
16110
16111        for (severity, vote, expect) in [
16112            (Severity::Major, ReviewVote::Reject, true),
16113            (Severity::Minor, ReviewVote::Reject, false),
16114            (Severity::Major, ReviewVote::Approve, false),
16115        ] {
16116            let mut round = review_round(false, 1, 1, 1, false, true);
16117            round.reviews = vec![ReviewRecord {
16118                reviewer: 1,
16119                agent: "alpha".to_owned(),
16120                summary: String::new(),
16121                findings: vec![Finding {
16122                    id: "R1-1-1".to_owned(),
16123                    severity,
16124                    file: None,
16125                    line: None,
16126                    title: "t".to_owned(),
16127                    detail: String::new(),
16128                }],
16129                vote: Some(vote),
16130                failed: None,
16131                duration_ms: 0,
16132                attempts: 0,
16133            }];
16134            let mut state = RunState::new(
16135                repo.clone(),
16136                "main".to_owned(),
16137                "deadbeef".to_owned(),
16138                "task".to_owned(),
16139                Config::default(),
16140            );
16141            state.reviews = vec![round];
16142            let mut runner = Runner {
16143                state,
16144                roles: ResolvedRoles {
16145                    implementers: Vec::new(),
16146                    judges: Vec::new(),
16147                    reviewers: Vec::new(),
16148                    fixer: None,
16149                    conductor: conductor(),
16150                    implementer_roster: Vec::new(),
16151                    judge_roster: Vec::new(),
16152                    reviewer_roster: Vec::new(),
16153                },
16154                sem: Arc::new(Semaphore::new(1)),
16155                pause: Pause::new(),
16156                interrupt: Pause::new(),
16157            };
16158            let shell = runner.state.config.shell();
16159            runner
16160                .stop_reviewing("round budget spent", &shell, &repo)
16161                .await
16162                .expect("stop_reviewing");
16163            assert_eq!(runner.state.status, RunStatus::Gating);
16164            assert_eq!(
16165                runner.state.contested_handoff.is_some(),
16166                expect,
16167                "{severity:?} + {vote:?}"
16168            );
16169        }
16170    }
16171
16172    /// The shape the incident this whole fix responds to actually had: the
16173    /// round budget spent, the last round's own e2e blocked on the shared
16174    /// build cache (held here by a live pid — this test process — exactly
16175    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16176    /// without forking a process). `stop_reviewing` must retry it — not
16177    /// silently leave the round looking untouched (the catch-up-only half of
16178    /// the bug), and not read the contention as a red `e2e` and block the
16179    /// run on it (the other half). Called directly, the same way
16180    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16181    /// above exercises `gate`, so this never needs a real cargo build to
16182    /// reach: the lease is never released, so `with_cache_lease` never gets
16183    /// past acquiring it into anything that would need a real workspace.
16184    #[tokio::test]
16185    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16186        crate::run::pin_test_home();
16187        let home = crate::run::home();
16188
16189        let tmp = tempfile::tempdir().expect("tempdir");
16190        let repo = tmp.path().join("repo");
16191        std::fs::create_dir_all(&repo).unwrap();
16192        init_repo(&repo);
16193        let head = crate::git::rev_parse(&repo, "HEAD")
16194            .await
16195            .expect("rev-parse");
16196        // Unique to this test, so holding its lease cannot collide with
16197        // another test sharing the same process-wide `home`.
16198        let cache_dir = tmp.path().join("target");
16199
16200        let mut config = Config::default();
16201        config.verify.e2e = vec![format!(
16202            "CARGO_TARGET_DIR='{}' test -f README.md",
16203            cache_dir.display()
16204        )];
16205        config.graph.review_rounds = 1;
16206        // Bounded so a regression that does start waiting fails the test in
16207        // seconds, not hangs it.
16208        config.graph.timeout_verify = Some(2);
16209        config.graph.timeout_resource_wait = 2;
16210
16211        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16212        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16213            .expect("no io error acquiring directly")
16214        {
16215            crate::cache::AcquireOutcome::Acquired(g) => g,
16216            crate::cache::AcquireOutcome::Busy(b) => {
16217                panic!("expected the direct acquire to win the lease first: {b:?}")
16218            }
16219        };
16220
16221        let mut state = RunState::new(
16222            repo.clone(),
16223            "main".to_owned(),
16224            head.clone(),
16225            "task".to_owned(),
16226            config,
16227        );
16228        state.candidates = vec![Candidate {
16229            index: 0,
16230            label: 'A',
16231            agent: "alpha".to_owned(),
16232            branch: "does-not-exist".to_owned(),
16233            worktree: repo.clone(),
16234            summary: String::new(),
16235            stat: String::new(),
16236            files: 0,
16237            commits: 0,
16238            empty: false,
16239            failed: None,
16240            verified_noop: None,
16241            duration_ms: 0,
16242            folded: false,
16243        }];
16244        state.tally = Some(Tally {
16245            first_choice: BTreeMap::from([('A', 1)]),
16246            borda: BTreeMap::new(),
16247            winner: 'A',
16248            rankings: 1,
16249            unanimous_initial: true,
16250            deliberated: false,
16251            changed_votes: 0,
16252            unanimous_final: true,
16253            tie_break: None,
16254            judges: 0,
16255            present: 0,
16256            quorum: 0,
16257            met_quorum: true,
16258            uncontested: Some("only candidate A produced a change".to_owned()),
16259        });
16260        // The round budget's last round, deferred: `needs_catchup_run`'s
16261        // other trigger. `stop_reviewing`'s retry machinery must treat this
16262        // exactly like a resource-blocked attempt once it actually runs.
16263        state.reviews = vec![ReviewRound {
16264            round: 1,
16265            head: head.clone(),
16266            verified_head: None,
16267            verified_at: None,
16268            reviews: Vec::new(),
16269            e2e: Vec::new(),
16270            fix: None,
16271            blocking: 1,
16272            answered: 1,
16273            expected: 1,
16274            clean: false,
16275            verify_retried: false,
16276            e2e_deferred: true,
16277            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16278            progressed: false,
16279            vote_split: false,
16280            reconsideration: Vec::new(),
16281            verdict: None,
16282        }];
16283
16284        let mut runner = Runner {
16285            state,
16286            roles: ResolvedRoles {
16287                implementers: Vec::new(),
16288                judges: Vec::new(),
16289                reviewers: Vec::new(),
16290                fixer: None,
16291                conductor: conductor(),
16292                implementer_roster: Vec::new(),
16293                judge_roster: Vec::new(),
16294                reviewer_roster: Vec::new(),
16295            },
16296            sem: Arc::new(Semaphore::new(1)),
16297            pause: Pause::new(),
16298            interrupt: Pause::new(),
16299        };
16300
16301        let shell = runner.state.config.shell();
16302        runner
16303            .stop_reviewing("round budget spent", &shell, &repo)
16304            .await
16305            .expect("stop_reviewing");
16306
16307        let last = runner.state.reviews.last().expect("round record");
16308        assert_eq!(
16309            last.e2e_status(),
16310            E2eStatus::ResourceBlocked,
16311            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16312             failed: {last:?}"
16313        );
16314        assert_eq!(
16315            last.verified_head.as_deref(),
16316            Some(head.as_str()),
16317            "which commit this attempt targeted is known even though nothing finished checking \
16318             it"
16319        );
16320        let first_attempt_at = last
16321            .verified_at
16322            .expect("when this attempt ran is known too");
16323        assert_ne!(
16324            runner.state.status,
16325            RunStatus::Blocked,
16326            "contention is evidence about the machine, not the patch — it must not settle the \
16327             run as blocked: {:?}",
16328            runner.state.status
16329        );
16330        assert!(
16331            !runner
16332                .state
16333                .events
16334                .iter()
16335                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16336            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16337            runner.state.events
16338        );
16339
16340        // The cache is still held: a later reentry must retry the same
16341        // round's verification again — not leave it looking exactly as
16342        // untouched as the first blocked attempt, which is indistinguishable
16343        // from never having tried again at all.
16344        runner
16345            .stop_reviewing("round budget spent", &shell, &repo)
16346            .await
16347            .expect("stop_reviewing retry");
16348        assert_eq!(
16349            runner.state.reviews.len(),
16350            1,
16351            "no new round was started: {:?}",
16352            runner.state.reviews
16353        );
16354        let last = runner.state.reviews.last().expect("round record");
16355        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16356        assert!(
16357            last.verified_at.expect("still known") > first_attempt_at,
16358            "a second reentry must be a fresh attempt, not a stale copy of the first"
16359        );
16360        assert_ne!(runner.state.status, RunStatus::Blocked);
16361
16362        held.release();
16363    }
16364
16365    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
16366    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
16367    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
16368    /// own top-of-function fast path (`review_conclusion`) correctly reads
16369    /// this shape as `None` rather than guessing `Blocked`, and the loop's
16370    /// own `for` range is empty once the round budget is spent, so
16371    /// `review_loop` must retry the check itself rather than silently doing
16372    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
16373    /// above exercises directly, but through `review_loop`'s own entry point
16374    /// this time, proving the wiring between the two rather than just the
16375    /// retry logic in isolation.
16376    #[tokio::test]
16377    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16378        crate::run::pin_test_home();
16379        let home = crate::run::home();
16380
16381        let tmp = tempfile::tempdir().expect("tempdir");
16382        let repo = tmp.path().join("repo");
16383        std::fs::create_dir_all(&repo).unwrap();
16384        init_repo(&repo);
16385        let head = crate::git::rev_parse(&repo, "HEAD")
16386            .await
16387            .expect("rev-parse");
16388        let cache_dir = tmp.path().join("target");
16389
16390        let mut config = Config::default();
16391        config.verify.e2e = vec![format!(
16392            "CARGO_TARGET_DIR='{}' test -f README.md",
16393            cache_dir.display()
16394        )];
16395        config.graph.review_rounds = 1;
16396        config.graph.timeout_verify = Some(2);
16397        config.graph.timeout_resource_wait = 2;
16398
16399        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16400        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16401            .expect("no io error acquiring directly")
16402        {
16403            crate::cache::AcquireOutcome::Acquired(g) => g,
16404            crate::cache::AcquireOutcome::Busy(b) => {
16405                panic!("expected the direct acquire to win the lease first: {b:?}")
16406            }
16407        };
16408
16409        let mut state = RunState::new(
16410            repo.clone(),
16411            "main".to_owned(),
16412            head.clone(),
16413            "task".to_owned(),
16414            config,
16415        );
16416        state.candidates = vec![Candidate {
16417            index: 0,
16418            label: 'A',
16419            agent: "alpha".to_owned(),
16420            branch: "does-not-exist".to_owned(),
16421            worktree: repo.clone(),
16422            summary: String::new(),
16423            stat: String::new(),
16424            files: 0,
16425            commits: 0,
16426            empty: false,
16427            failed: None,
16428            verified_noop: None,
16429            duration_ms: 0,
16430            folded: false,
16431        }];
16432        state.tally = Some(Tally {
16433            first_choice: BTreeMap::from([('A', 1)]),
16434            borda: BTreeMap::new(),
16435            winner: 'A',
16436            rankings: 1,
16437            unanimous_initial: true,
16438            deliberated: false,
16439            changed_votes: 0,
16440            unanimous_final: true,
16441            tie_break: None,
16442            judges: 0,
16443            present: 0,
16444            quorum: 0,
16445            met_quorum: true,
16446            uncontested: Some("only candidate A produced a change".to_owned()),
16447        });
16448        // The exact shape a prior process's `stop_reviewing` would have left
16449        // on disk: the round budget's last round, a real attempt already
16450        // made and already resource-blocked.
16451        state.reviews = vec![ReviewRound {
16452            round: 1,
16453            head: head.clone(),
16454            verified_head: Some(head.clone()),
16455            verified_at: Some(jiff::Timestamp::now()),
16456            reviews: Vec::new(),
16457            e2e: vec![CommandOutcome {
16458                command: format!(
16459                    "CARGO_TARGET_DIR='{}' test -f README.md",
16460                    cache_dir.display()
16461                ),
16462                code: None,
16463                output_tail: "waiting for the shared build cache".to_owned(),
16464                duration_ms: 0,
16465                resource_blocked: true,
16466            }],
16467            fix: None,
16468            blocking: 1,
16469            answered: 1,
16470            expected: 1,
16471            clean: false,
16472            verify_retried: false,
16473            e2e_deferred: false,
16474            e2e_defer_reason: None,
16475            progressed: false,
16476            vote_split: false,
16477            reconsideration: Vec::new(),
16478            verdict: None,
16479        }];
16480
16481        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16482        let mut runner = Runner {
16483            state,
16484            roles: ResolvedRoles {
16485                implementers: Vec::new(),
16486                judges: Vec::new(),
16487                reviewers: Vec::new(),
16488                fixer: None,
16489                conductor: conductor(),
16490                implementer_roster: Vec::new(),
16491                judge_roster: Vec::new(),
16492                reviewer_roster: Vec::new(),
16493            },
16494            sem: Arc::new(Semaphore::new(1)),
16495            pause: Pause::new(),
16496            interrupt: Pause::new(),
16497        };
16498
16499        // The lease is still held throughout, so this reentry's own retry is
16500        // also contended — proving `review_loop` actually tried again (not
16501        // that it happened to succeed) is what the timestamp comparison
16502        // below is for.
16503        runner.review_loop().await.expect("review_loop");
16504
16505        assert_eq!(
16506            runner.state.reviews.len(),
16507            1,
16508            "no new round was started on top of the unresolved one: {:?}",
16509            runner.state.reviews
16510        );
16511        let last = &runner.state.reviews[0];
16512        assert_eq!(
16513            last.e2e_status(),
16514            E2eStatus::ResourceBlocked,
16515            "still contended: {last:?}"
16516        );
16517        assert!(
16518            last.verified_at.expect("still known") > first_attempt_at,
16519            "review_loop must have actually retried the check, not left it exactly as found"
16520        );
16521        assert_ne!(
16522            runner.state.status,
16523            RunStatus::Blocked,
16524            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16525            runner.state.status
16526        );
16527
16528        held.release();
16529    }
16530
16531    #[tokio::test]
16532    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16533        crate::run::pin_test_home();
16534        for status in [RunStatus::Blocked, RunStatus::Gating] {
16535            let mut runner = runner_at(status);
16536            runner.state.config.graph.land = true;
16537            runner.state.gate_ran = true;
16538            runner.state.merge = Some(MergeOutcome {
16539                mode: MergeMode::Pr,
16540                ok: false,
16541                detail: "the fixer produced no commit".into(),
16542                empty: false,
16543            });
16544            let url = "https://example.invalid/o/r/pull/1";
16545            runner.state.pr = Some(crate::run::PrRecord {
16546                url: url.into(),
16547                number: 1,
16548                state: "open".into(),
16549                checks: "red".into(),
16550                round: 1,
16551                rounds: 1,
16552                red_at_merge: Vec::new(),
16553            });
16554            runner.state.save().unwrap();
16555            // The nonexistent repository makes observation fail before a forge
16556            // command can run. We still must reach land, rather than silently
16557            // return from the graph with its old gate and merge records.
16558            runner.execute_graph().await.unwrap();
16559            assert_eq!(runner.state.status, RunStatus::Blocked);
16560            assert!(runner.state.land_sync_pending);
16561            assert!(runner.state.events.iter().any(|event|
16562                event.node == "land" && event.message == format!("watching {url}")));
16563            assert_eq!(
16564                runner.state.merge.as_ref().unwrap().detail,
16565                "the fixer produced no commit"
16566            );
16567        }
16568    }
16569
16570    #[tokio::test]
16571    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16572        crate::run::pin_test_home();
16573        let tmp = tempfile::tempdir().expect("tempdir");
16574        let repo = tmp.path().join("repo");
16575        std::fs::create_dir_all(&repo).unwrap();
16576        init_repo(&repo);
16577
16578        let mut config = Config::default();
16579        config.merge.mode = MergeMode::Pr;
16580        config.graph.land = true;
16581        config.graph.land_approval = false;
16582
16583        let mut state = RunState::new(
16584            repo.clone(),
16585            "main".to_owned(),
16586            "deadbeef".to_owned(),
16587            "task".to_owned(),
16588            config,
16589        );
16590        state.candidates = vec![Candidate {
16591            index: 0,
16592            label: 'A',
16593            agent: "alpha".to_owned(),
16594            branch: "does-not-exist".to_owned(),
16595            worktree: repo.clone(),
16596            summary: String::new(),
16597            stat: String::new(),
16598            files: 0,
16599            commits: 0,
16600            empty: false,
16601            failed: None,
16602            verified_noop: None,
16603            duration_ms: 0,
16604            folded: false,
16605        }];
16606        state.tally = Some(Tally {
16607            first_choice: BTreeMap::from([('A', 1)]),
16608            borda: BTreeMap::new(),
16609            winner: 'A',
16610            rankings: 1,
16611            unanimous_initial: true,
16612            deliberated: false,
16613            changed_votes: 0,
16614            unanimous_final: true,
16615            tie_break: None,
16616            judges: 0,
16617            present: 0,
16618            quorum: 0,
16619            met_quorum: true,
16620            uncontested: Some("only candidate A produced a change".to_owned()),
16621        });
16622        state.reviews = vec![ReviewRound {
16623            round: 1,
16624            head: "deadbeef".to_owned(),
16625            verified_head: None,
16626            verified_at: None,
16627            reviews: Vec::new(),
16628            e2e: Vec::new(),
16629            fix: None,
16630            blocking: 0,
16631            answered: 0,
16632            expected: 0,
16633            clean: true,
16634            verify_retried: false,
16635            e2e_deferred: false,
16636            e2e_defer_reason: None,
16637            progressed: false,
16638            vote_split: false,
16639            reconsideration: Vec::new(),
16640            verdict: None,
16641        }];
16642        state.gate = vec![CommandOutcome {
16643            command: "test".to_owned(),
16644            code: Some(0),
16645            output_tail: String::new(),
16646            duration_ms: 0,
16647            resource_blocked: false,
16648        }];
16649        state.gate_ran = true;
16650        // A first pass through `merge` already pushed and opened this pull
16651        // request; `status` is `Landing` because a previous call into `land`
16652        // parked or was interrupted before it reached a terminal outcome.
16653        state.status = RunStatus::Landing;
16654        state.merge = Some(MergeOutcome {
16655            mode: MergeMode::Pr,
16656            ok: true,
16657            detail: "https://example.invalid/x/y/pull/1".to_owned(),
16658            empty: false,
16659        });
16660
16661        // The Landing-resume shortcut calls `run_land` directly rather than
16662        // through `merge`, which is exactly the call site that used to skip
16663        // `settle_questions` - see the fixture below.
16664        ask_test_home();
16665        // A real resume loads a persisted run; driver ownership is checked
16666        // against that record before the Landing shortcut is entered.
16667        state.save().expect("save landing run");
16668        let store = ask::Questions::open();
16669        let q = ask_open_question(&store, &state.id);
16670
16671        let mut runner = Runner {
16672            state,
16673            roles: ResolvedRoles {
16674                implementers: Vec::new(),
16675                judges: Vec::new(),
16676                reviewers: Vec::new(),
16677                fixer: None,
16678                conductor: conductor(),
16679                implementer_roster: Vec::new(),
16680                judge_roster: Vec::new(),
16681                reviewer_roster: Vec::new(),
16682            },
16683            sem: Arc::new(Semaphore::new(1)),
16684            pause: Pause::new(),
16685            interrupt: Pause::new(),
16686        };
16687
16688        // `execute`, not `merge` directly: the Landing-resume shortcut lives
16689        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
16690        // exactly because `review_loop` would otherwise clobber the marker
16691        // first.
16692        runner.execute().await.expect("execute");
16693
16694        assert_eq!(
16695            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16696            Some("https://example.invalid/x/y/pull/1"),
16697            "reentry must not push again or open a second pull request over the \
16698             one `land` is already watching"
16699        );
16700        assert_ne!(
16701            runner.state.status,
16702            RunStatus::Landing,
16703            "land could not actually reach the fake pull request, so it must \
16704             have given up rather than left the run silently parked forever"
16705        );
16706        // `land` could not reach the fake pull request, so it gave up into
16707        // `Blocked` - still resumable, so the question must not have been
16708        // swept just because this branch now also calls `settle_questions`.
16709        assert_eq!(runner.state.status, RunStatus::Blocked);
16710        assert!(
16711            store.get(&q.id).unwrap().status.open(),
16712            "Blocked is still alive; settle_questions must have been a no-op here"
16713        );
16714    }
16715
16716    fn state_with_round(round: ReviewRound) -> RunState {
16717        let mut s = RunState::new(
16718            PathBuf::from("/repo"),
16719            "main".to_owned(),
16720            "abc1234".to_owned(),
16721            "add retries".to_owned(),
16722            Config::default(),
16723        );
16724        s.reviews = vec![round];
16725        s
16726    }
16727
16728    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16729        crate::verdict::Finding {
16730            id: id.to_owned(),
16731            severity,
16732            file: None,
16733            line: None,
16734            title: title.to_owned(),
16735            detail: String::new(),
16736        }
16737    }
16738
16739    #[test]
16740    fn pr_body_names_open_findings_and_declined_ones() {
16741        let round = ReviewRound {
16742            round: 2,
16743            head: "deadbee".to_owned(),
16744            verified_head: None,
16745            verified_at: None,
16746            reviews: vec![ReviewRecord {
16747                attempts: 0,
16748                reviewer: 1,
16749                agent: "alpha".to_owned(),
16750                summary: String::new(),
16751                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16752                vote: None,
16753                failed: None,
16754                duration_ms: 0,
16755            }],
16756            e2e: vec![CommandOutcome {
16757                command: "cargo test".to_owned(),
16758                code: Some(0),
16759                output_tail: String::new(),
16760                duration_ms: 0,
16761                resource_blocked: false,
16762            }],
16763            verify_retried: false,
16764            e2e_deferred: false,
16765            e2e_defer_reason: None,
16766            fix: Some(FixRecord {
16767                agent: "alpha".to_owned(),
16768                addressed: Vec::new(),
16769                rejected: vec![crate::verdict::Rejection {
16770                    id: "R1-1-1".to_owned(),
16771                    why: "not reachable from any caller".to_owned(),
16772                }],
16773                notes: String::new(),
16774                committed: true,
16775                failed: None,
16776                duration_ms: 0,
16777                continuation: None,
16778            }),
16779            blocking: 0,
16780            answered: 1,
16781            expected: 1,
16782            clean: false,
16783            progressed: true,
16784            vote_split: false,
16785            reconsideration: Vec::new(),
16786            verdict: None,
16787        };
16788        let state = state_with_round(round);
16789        let body = pr_message(&state, 'A').body;
16790
16791        assert!(body.contains("add retries"), "the task must still be there");
16792        assert!(body.contains("R2-1-1"), "{body}");
16793        assert!(!body.contains("unused import"), "{body}");
16794        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16795        assert!(
16796            !body.contains("not reachable from any caller"),
16797            "the reason it was declined: {body}"
16798        );
16799    }
16800
16801    #[test]
16802    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16803        let round = ReviewRound {
16804            round: 1,
16805            head: "deadbee".to_owned(),
16806            verified_head: None,
16807            verified_at: None,
16808            reviews: vec![ReviewRecord {
16809                attempts: 0,
16810                reviewer: 1,
16811                agent: "alpha".to_owned(),
16812                summary: String::new(),
16813                findings: Vec::new(),
16814                vote: None,
16815                failed: None,
16816                duration_ms: 0,
16817            }],
16818            e2e: Vec::new(),
16819            verify_retried: false,
16820            e2e_deferred: false,
16821            e2e_defer_reason: None,
16822            fix: None,
16823            blocking: 0,
16824            answered: 1,
16825            expected: 1,
16826            clean: true,
16827            progressed: false,
16828            vote_split: false,
16829            reconsideration: Vec::new(),
16830            verdict: None,
16831        };
16832        let state = state_with_round(round);
16833        let body = pr_message(&state, 'A').body;
16834        assert!(!body.contains("Open review findings"), "{body}");
16835        assert!(!body.contains("Declined"), "{body}");
16836    }
16837
16838    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16839        let mut state = RunState::new(
16840            PathBuf::from("/repo"),
16841            "main".to_owned(),
16842            "abc1234".to_owned(),
16843            instruction.to_owned(),
16844            Config::default(),
16845        );
16846        state.candidates.push(Candidate {
16847            index: 0,
16848            label: 'A',
16849            agent: "alpha".to_owned(),
16850            branch: "magi/x/A".to_owned(),
16851            worktree: PathBuf::from("/wt"),
16852            summary: summary.to_owned(),
16853            stat: String::new(),
16854            files: 1,
16855            commits: 1,
16856            empty: false,
16857            failed: None,
16858            verified_noop: None,
16859            folded: false,
16860            duration_ms: 0,
16861        });
16862        state
16863    }
16864
16865    fn review_state(subjects: &[&str]) -> RunState {
16866        let mut state = state_with_summary(
16867            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16868            "",
16869        );
16870        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16871        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16872        state
16873    }
16874
16875    /// A branch rebased onto a main that moved past the recorded
16876    /// `base_commit` is titled from its own first commit, never main's.
16877    #[tokio::test]
16878    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16879        ask_test_home();
16880        let tmp = tempfile::tempdir().unwrap();
16881        let repo = tmp.path().join("repo");
16882        std::fs::create_dir_all(&repo).unwrap();
16883        init_repo(&repo);
16884        let origin = tmp.path().join("origin.git");
16885        let g = |dir: &Path, args: &[&str]| {
16886            let out = std::process::Command::new("git")
16887                .args(args)
16888                .current_dir(dir)
16889                .quiet()
16890                .output()
16891                .expect("spawn git");
16892            assert!(
16893                out.status.success(),
16894                "git {args:?}: {}",
16895                String::from_utf8_lossy(&out.stderr)
16896            );
16897        };
16898        g(
16899            tmp.path(),
16900            &[
16901                "clone",
16902                "--bare",
16903                "-q",
16904                repo.to_str().unwrap(),
16905                origin.to_str().unwrap(),
16906            ],
16907        );
16908        g(
16909            &repo,
16910            &["remote", "add", "origin", origin.to_str().unwrap()],
16911        );
16912        let c1 = git::rev_parse(&repo, "main").await.unwrap();
16913
16914        // Main moves on; the branch is built on top of the new main.
16915        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16916        g(&repo, &["add", "-A"]);
16917        g(
16918            &repo,
16919            &["commit", "-q", "-m", "chore(deps): update a crate"],
16920        );
16921        g(&repo, &["push", "-q", "origin", "main"]);
16922        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16923        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16924        g(&repo, &["add", "-A"]);
16925        g(
16926            &repo,
16927            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16928        );
16929        g(&repo, &["checkout", "-q", "main"]);
16930
16931        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16932        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16933        // Without a readable tracking ref the recorded base's merge base is used.
16934        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16935        assert_eq!(fallback, c1);
16936
16937        let mut state = review_state(&[
16938            "chore(deps): update a crate",
16939            "fix(daemon): apply a chosen action",
16940        ]);
16941        state.repo = repo.clone();
16942        state.base_branch = "main".to_owned();
16943        state.base_commit = c1;
16944        refresh_reviewed_commits(&mut state, "feat/own").await;
16945        assert_eq!(
16946            state.reviewed_commits,
16947            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16948        );
16949        assert_eq!(
16950            review_title(&state).as_deref(),
16951            Some("fix(daemon): apply a chosen action")
16952        );
16953        assert_eq!(
16954            leaked_subjects(&state, "feat/own").await,
16955            Some(vec!["chore(deps): update a crate".to_owned()])
16956        );
16957        // A stale tracking ref is still used when the fetch fails, but the
16958        // leak list is withheld.
16959        state.config.merge.remote = "nowhere".to_owned();
16960        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16961    }
16962
16963    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
16964    /// with `origin/main` one commit behind it, and a summarizing agent that
16965    /// prints whatever `reply` holds.
16966    struct SummaryFixture {
16967        _tmp: tempfile::TempDir,
16968        repo: PathBuf,
16969        reply: PathBuf,
16970        runner: Runner,
16971    }
16972
16973    fn summary_fixture() -> SummaryFixture {
16974        ask_test_home();
16975        let tmp = tempfile::tempdir().unwrap();
16976        let repo = tmp.path().join("repo");
16977        std::fs::create_dir_all(&repo).unwrap();
16978        init_repo(&repo);
16979        let origin = tmp.path().join("origin.git");
16980        let g = |args: &[&str]| {
16981            let out = std::process::Command::new("git")
16982                .args(args)
16983                .current_dir(&repo)
16984                .quiet()
16985                .output()
16986                .expect("spawn git");
16987            assert!(
16988                out.status.success(),
16989                "git {args:?}: {}",
16990                String::from_utf8_lossy(&out.stderr)
16991            );
16992        };
16993        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16994        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16995        g(&["push", "-q", "origin", "main"]);
16996        g(&["checkout", "-q", "-b", "feat/own"]);
16997        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16998        g(&["add", "-A"]);
16999        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17000        let reply = tmp.path().join("reply.txt");
17001        std::fs::write(
17002            &reply,
17003            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17004        )
17005        .unwrap();
17006        let mut runner = runner_at(RunStatus::Gating);
17007        let mut state = review_state(&["feat(own): add the own file"]);
17008        state.repo = repo.clone();
17009        state.base_branch = "main".to_owned();
17010        state.candidates[0].branch = "feat/own".to_owned();
17011        state.candidates[0].worktree = repo.clone();
17012        let mut author = spec("alpha");
17013        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17014        author
17015            .env
17016            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17017        state.config.agents = vec![author];
17018        runner.state = state;
17019        SummaryFixture {
17020            _tmp: tmp,
17021            repo,
17022            reply,
17023            runner,
17024        }
17025    }
17026
17027    async fn posted(f: &mut SummaryFixture) -> PrMessage {
17028        let winner = f.runner.state.candidates[0].clone();
17029        f.runner
17030            .guarded_pr_message(&winner, None, true)
17031            .await
17032            .unwrap()
17033            .expect("an English description never parks")
17034    }
17035
17036    fn clean_round_with_minor_findings() -> ReviewRound {
17037        let mut round = passing_round(2, None);
17038        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17039        round.reviews = vec![ReviewRecord {
17040            attempts: 0,
17041            reviewer: 1,
17042            agent: "alpha".into(),
17043            summary: "内部レビュー".into(),
17044            findings: vec![
17045                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17046                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17047            ],
17048            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17049            failed: None,
17050            duration_ms: 0,
17051        }];
17052        round
17053    }
17054
17055    #[tokio::test]
17056    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17057        for review_only in [false, true] {
17058            let mut f = summary_fixture();
17059            if !review_only {
17060                f.runner.state.candidates[0].agent = "alpha".into();
17061            }
17062            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17063            assert!(f.runner.state.reviews[0].clean);
17064            assert!(f.runner.state.reviews[0].fix.is_none());
17065            assert!(f.runner.state.open_findings().is_empty());
17066            assert_eq!(
17067                crate::pr_outcome::required_finding_ids(&f.runner.state),
17068                ["R2-1-1", "R2-1-2"]
17069            );
17070            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17071            let evidence = crate::pr_outcome::summary_evidence(
17072                &f.runner.state,
17073                crate::pr_outcome::FinalHead::At(&head),
17074            );
17075            assert!(
17076                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17077            );
17078            // The ordinary reply omits both IDs: no cached description is adopted.
17079            let omitted = posted(&mut f).await;
17080            assert!(omitted.body.contains("No description"));
17081            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17082            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17083            assert!(f.runner.state.change_summary.is_none());
17084            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17085            let described = posted(&mut f).await;
17086            assert!(described.body.contains("improve documentation for callers"));
17087            assert!(described.body.contains("remaining edge case"));
17088            assert!(described.body.contains("## Open review findings"));
17089            assert!(!described.body.contains("未修正"));
17090        }
17091    }
17092
17093    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17094        let mut f = summary_fixture();
17095        if !review_only {
17096            f.runner.state.candidates[0].agent = "alpha".into();
17097        }
17098        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17099        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17100        fixed
17101            .fix
17102            .as_mut()
17103            .unwrap()
17104            .rejected
17105            .push(crate::verdict::Rejection {
17106                id: "R1-1-1".into(),
17107                why: "呼び出されないコードのため辞退".into(),
17108            });
17109        let mut clean = clean_round_with_minor_findings();
17110        clean.verified_head = Some(head.clone());
17111        f.runner.state.reviews = vec![fixed, clean];
17112        ok_gate(&mut f.runner.state, Some(&head));
17113        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17114        f.runner.state.config.agents.push(scripted("beta", reply));
17115        f.runner.state.config.roles.language_judge =
17116            Some(crate::config::AgentChoice::One("beta".into()));
17117        f
17118    }
17119
17120    #[tokio::test]
17121    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17122        for review_only in [false, true] {
17123            let mut f = rewrite_fixture_with_findings(review_only,
17124                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17125            let m = posted(&mut f).await;
17126            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17127            let expected =
17128                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17129            assert!(m.body.contains(&expected));
17130            assert!(m.body.contains("## Review rounds"));
17131            assert!(
17132                m.body
17133                    .contains("Gate (`verify.gate`): passed on the final commit")
17134            );
17135            assert!(
17136                m.body
17137                    .contains("Tests (`verify.e2e`): passed on the final commit")
17138            );
17139            let c = f.runner.state.change_summary.as_ref().unwrap();
17140            assert!(
17141                m.body
17142                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17143            );
17144            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17145            assert!(crate::pr_outcome::covers_required_findings(
17146                &f.runner.state,
17147                &m.body
17148            ));
17149            assert!(m.body.contains("documentation remains incomplete"));
17150            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17151            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17152            assert!(f.runner.state.github_text.is_none());
17153        }
17154    }
17155
17156    #[tokio::test]
17157    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17158        for review_only in [false, true] {
17159            for reply in [
17160                "TITLE: fix: public description\nA short English description.",
17161                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17162            ] {
17163                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17164                let winner = f.runner.state.candidates[0].clone();
17165                let posted = f
17166                    .runner
17167                    .guarded_pr_message(&winner, None, true)
17168                    .await
17169                    .unwrap();
17170                assert!(posted.is_none());
17171                assert!(f.runner.state.parked);
17172                let withheld = f.runner.state.github_text.as_ref().unwrap();
17173                assert!(
17174                    withheld.rewrite.is_none(),
17175                    "an incomplete rewrite must not be selectable"
17176                );
17177                let q = ask::Questions::open()
17178                    .get(withheld.question.as_ref().unwrap())
17179                    .unwrap();
17180                assert!(
17181                    !q.choices
17182                        .contains(&crate::github_text::USE_REWRITE.to_owned())
17183                );
17184                assert!(
17185                    q.detail.contains("R1-1-1")
17186                        && q.detail.contains("R2-1-1")
17187                        && q.detail.contains("R2-1-2")
17188                );
17189            }
17190        }
17191    }
17192
17193    #[tokio::test]
17194    async fn japanese_reports_stay_local_in_both_public_run_modes() {
17195        for review_only in [false, true] {
17196            let mut f = summary_fixture();
17197            if !review_only {
17198                f.runner.state.candidates[0].agent = "alpha".into();
17199            }
17200            f.runner.state.instruction = "内部タスクの説明".into();
17201            f.runner.state.candidates[0].summary = "内部の候補報告".into();
17202            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17203            f.runner.state.reviews = vec![
17204                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17205                round_with_notes(2, Some("二回目の内部修正報告です。")),
17206                passing_round(3, Some(&head)),
17207            ];
17208            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17209            ok_gate(&mut f.runner.state, Some(&head));
17210            let captured = f.repo.join("public-prompt.txt");
17211            let author = &mut f.runner.state.config.agents[0];
17212            author.command = vec![
17213                "sh".into(),
17214                "-c".into(),
17215                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17216            ];
17217            author.env.insert(
17218                "PROMPT_FILE".into(),
17219                captured.to_string_lossy().into_owned(),
17220            );
17221            let m = posted(&mut f).await;
17222            assert!(m.body.contains("Adds `own.txt`"));
17223            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17224            assert!(m.body.contains("R1-1-1"));
17225            assert!(
17226                m.body
17227                    .contains("Gate (`verify.gate`): passed on the final commit")
17228            );
17229            assert!(
17230                m.body
17231                    .contains("Tests (`verify.e2e`): passed on the final commit")
17232            );
17233            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17234            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17235            let prompt = std::fs::read_to_string(captured).unwrap();
17236            assert!(prompt.contains(&format!(
17237                "git diff {} {head}",
17238                f.runner.state.change_summary.as_ref().unwrap().base
17239            )));
17240            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17241            assert_eq!(
17242                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17243                "二回目の内部修正報告です。"
17244            );
17245        }
17246    }
17247
17248    #[tokio::test]
17249    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17250        for review_only in [false, true] {
17251            let mut f = summary_fixture();
17252            if !review_only {
17253                f.runner.state.candidates[0].agent = "alpha".into();
17254            }
17255            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17256            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17257            std::fs::write(&f.reply, "no usable description").unwrap();
17258            let failed = posted(&mut f).await;
17259            assert!(
17260                failed
17261                    .body
17262                    .contains("No description of what this change does could be generated")
17263            );
17264            assert!(failed.body.is_ascii());
17265            assert!(f.runner.state.github_text.is_none());
17266            assert!(f.runner.state.change_summary.is_none());
17267            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17268            let evidence = crate::pr_outcome::summary_evidence(
17269                &f.runner.state,
17270                crate::pr_outcome::FinalHead::At(&head),
17271            );
17272            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17273            assert!(
17274                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17275                    .exists()
17276            );
17277            f.runner.state =
17278                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17279            std::fs::write(
17280                &f.reply,
17281                "TITLE: feat(own): add the own file\nAdds the committed own file.",
17282            )
17283            .unwrap();
17284            let retried = posted(&mut f).await;
17285            assert!(retried.body.contains("Adds the committed own file."));
17286            assert_eq!(
17287                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17288                "日本語の修正報告"
17289            );
17290        }
17291    }
17292
17293    #[tokio::test]
17294    async fn same_head_verification_and_findings_invalidate_public_prose() {
17295        let mut f = summary_fixture();
17296        posted(&mut f).await;
17297        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17298        let original = f
17299            .runner
17300            .state
17301            .change_summary
17302            .as_ref()
17303            .unwrap()
17304            .evidence
17305            .clone();
17306        ok_gate(&mut f.runner.state, Some(&head));
17307        std::fs::write(
17308            &f.reply,
17309            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17310        )
17311        .unwrap();
17312        let verified = posted(&mut f).await;
17313        assert!(verified.body.contains("now verified"));
17314        assert_ne!(
17315            f.runner.state.change_summary.as_ref().unwrap().evidence,
17316            original
17317        );
17318        let mut round = round_with_notes(1, Some("内部報告"));
17319        round.reviews.push(ReviewRecord {
17320            attempts: 0,
17321            reviewer: 1,
17322            agent: "alpha".into(),
17323            summary: "内部レビュー".into(),
17324            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17325            vote: None,
17326            failed: None,
17327            duration_ms: 0,
17328        });
17329        round
17330            .fix
17331            .as_mut()
17332            .unwrap()
17333            .rejected
17334            .push(crate::verdict::Rejection {
17335                id: "R1-1-1".into(),
17336                why: "呼び出し元から到達できない".into(),
17337            });
17338        f.runner.state.reviews = vec![round];
17339        // A description omitting the required ID is unusable, even with a title.
17340        let omitted = posted(&mut f).await;
17341        assert!(omitted.body.contains("No description"));
17342        assert!(omitted.body.contains("R1-1-1"));
17343        assert!(!omitted.body.contains("到達"));
17344        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17345        let described = posted(&mut f).await;
17346        assert!(described.body.contains("unreachable from callers"));
17347        assert!(described.body.contains("R1-1-1"));
17348        assert!(!described.body.contains("到達"));
17349        let evidence = f
17350            .runner
17351            .state
17352            .change_summary
17353            .as_ref()
17354            .unwrap()
17355            .evidence
17356            .clone();
17357        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17358        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17359        assert!(posted(&mut f).await.body.contains("rationale updated"));
17360        assert_ne!(
17361            f.runner.state.change_summary.as_ref().unwrap().evidence,
17362            evidence
17363        );
17364        // Old serialized summaries have no evidence fingerprint and must regenerate.
17365        f.runner
17366            .state
17367            .change_summary
17368            .as_mut()
17369            .unwrap()
17370            .evidence
17371            .clear();
17372        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17373        assert!(posted(&mut f).await.body.contains("current specification"));
17374    }
17375
17376    #[tokio::test]
17377    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17378        let mut f = summary_fixture();
17379        let m = posted(&mut f).await;
17380        assert_eq!(m.title, "feat(own): add the own file");
17381        assert!(
17382            m.body
17383                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17384            "{}",
17385            m.body
17386        );
17387        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17388        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17389        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17390        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17391        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17392        let kept = f.runner.state.change_summary.as_ref().unwrap();
17393        assert_eq!(
17394            (kept.head.as_str(), kept.base.as_str()),
17395            (head.as_str(), base.as_str())
17396        );
17397        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17398    }
17399
17400    #[tokio::test]
17401    async fn review_only_summary_receives_the_configured_github_language() {
17402        let mut f = summary_fixture();
17403        f.runner.state.config.graph.github_language = "ja".into();
17404        let captured = f.repo.join("summary-prompt.txt");
17405        let author = &mut f.runner.state.config.agents[0];
17406        author.command = vec![
17407            "sh".into(),
17408            "-c".into(),
17409            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17410        ];
17411        author.env.insert(
17412            "PROMPT_FILE".into(),
17413            captured.to_string_lossy().into_owned(),
17414        );
17415        std::fs::write(
17416            &f.reply,
17417            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17418        )
17419        .unwrap();
17420        let m = posted(&mut f).await;
17421        let prompt = std::fs::read_to_string(captured).unwrap();
17422        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17423        assert!(!prompt.contains("in English"));
17424        assert_eq!(m.title, "feat(own): ファイルを追加");
17425        assert!(m.body.contains("own.txt を追加します。"));
17426    }
17427
17428    #[tokio::test]
17429    async fn review_fixes_come_after_the_whole_change_summary() {
17430        let mut f = summary_fixture();
17431        f.runner.state.reviews = vec![
17432            round_with_notes(1, Some("handled the PATH case")),
17433            round_with_notes(2, None),
17434        ];
17435        let m = posted(&mut f).await;
17436        let summary = m.body.find("## Summary").expect("summary");
17437        let fixes = m.body.find("## Review rounds").expect("review rounds");
17438        assert!(summary < fixes, "{}", m.body);
17439        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17440        assert!(m.body.contains("## Final verification"), "{}", m.body);
17441    }
17442
17443    #[tokio::test]
17444    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17445        let mut f = summary_fixture();
17446        let first = posted(&mut f).await;
17447        let first_head = f.runner.state.change_summary.clone().unwrap().head;
17448        // The same head reuses the record: a different reply is not read.
17449        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17450        let again = posted(&mut f).await;
17451        assert_eq!((again.title, again.body), (first.title, first.body));
17452
17453        let g = |args: &[&str]| {
17454            let out = std::process::Command::new("git")
17455                .args(args)
17456                .current_dir(&f.repo)
17457                .quiet()
17458                .output()
17459                .expect("spawn git");
17460            assert!(out.status.success(), "git {args:?}");
17461        };
17462        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17463        g(&["add", "-A"]);
17464        g(&["commit", "-q", "-m", "fix(own): handle more"]);
17465        let second = posted(&mut f).await;
17466        let kept = f.runner.state.change_summary.clone().unwrap();
17467        assert_ne!(kept.head, first_head);
17468        assert!(second.body.contains("Not read yet."), "{}", second.body);
17469        assert!(!second.body.contains("so the first summary is read"));
17470        assert_eq!(second.title, "feat(own): other");
17471    }
17472
17473    #[tokio::test]
17474    async fn an_unavailable_summary_is_stated_and_never_invented() {
17475        let mut f = summary_fixture();
17476        std::fs::write(&f.reply, "no title line here").unwrap();
17477        let m = posted(&mut f).await;
17478        assert!(
17479            m.body
17480                .contains("No description of what this change does could be generated"),
17481            "{}",
17482            m.body
17483        );
17484        assert!(m.body.contains("internal reports remain in local records"));
17485        assert!(f.runner.state.change_summary.is_none());
17486        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17487        // A preview asks nobody and claims nothing.
17488        let winner = f.runner.state.candidates[0].clone();
17489        let preview = f
17490            .runner
17491            .guarded_pr_message(&winner, None, false)
17492            .await
17493            .unwrap()
17494            .unwrap();
17495        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17496    }
17497
17498    #[test]
17499    fn pr_message_review_single_commit_uses_its_subject() {
17500        let state = review_state(&["feat(nats): per-role user"]);
17501        let m = pr_message(&state, 'A');
17502        assert_eq!(m.title, "feat(nats): per-role user");
17503        assert!(!m.body.contains("Review the work already"), "{}", m.body);
17504        assert!(m.body.contains("## Commits under review"), "{}", m.body);
17505    }
17506
17507    #[test]
17508    fn pr_message_review_multi_commit_takes_the_oldest() {
17509        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17510        let m = pr_message(&state, 'A');
17511        assert_eq!(m.title, "feat: the change");
17512        for s in ["feat: the change", "fix: typo", "fix: again"] {
17513            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17514        }
17515    }
17516
17517    #[test]
17518    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17519        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17520            let mut state = review_state(&[first, "fix: later fixup"]);
17521            state.candidates[0].branch = "機能/ログイン".to_owned();
17522            let m = pr_message(&state, 'A');
17523            assert!(
17524                m.title.starts_with("chore: land candidate A of run"),
17525                "{}",
17526                m.title
17527            );
17528        }
17529    }
17530
17531    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17532        BranchFacts {
17533            commits: commits
17534                .iter()
17535                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17536                .collect(),
17537            stat: stat.to_owned(),
17538        }
17539    }
17540
17541    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17542        let mut r = review_round(true, 0, 1, 1, true, true);
17543        r.round = round;
17544        r.fix = notes.map(|n| FixRecord {
17545            agent: "fixer".to_owned(),
17546            addressed: Vec::new(),
17547            rejected: Vec::new(),
17548            notes: n.to_owned(),
17549            committed: true,
17550            failed: None,
17551            duration_ms: 0,
17552            continuation: None,
17553        });
17554        r
17555    }
17556
17557    #[test]
17558    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17559        let state = review_state(&["ignored"]);
17560        let f = facts(
17561            &[
17562                (
17563                    "fix(login): resolve PATH on macOS",
17564                    "Login shells skip rc files.",
17565                ),
17566                ("fix: address review", ""),
17567            ],
17568            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17569        );
17570        let m = pr_message_with(&state, 'A', Some(&f));
17571        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17572        assert!(
17573            m.body
17574                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
17575            "{}",
17576            m.body
17577        );
17578        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17579        assert!(m.body.contains("## Diff stat"), "{}", m.body);
17580        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17581        for banned in [
17582            "Review the work already",
17583            "no task statement",
17584            "Original task",
17585        ] {
17586            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17587        }
17588        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17589    }
17590
17591    #[test]
17592    fn pr_message_review_truncates_a_huge_first_commit_body() {
17593        let state = review_state(&["ignored"]);
17594        let f = facts(
17595            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17596            "s",
17597        );
17598        let m = pr_message_with(&state, 'A', Some(&f));
17599        assert!(m.body.len() < 30_000, "{}", m.body.len());
17600        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17601        assert!(
17602            m.body.contains("- ... 1 more commit(s)"),
17603            "{}",
17604            m.body.len()
17605        );
17606        assert!(m.body.ends_with("magi:candidate-a\n"));
17607    }
17608
17609    #[test]
17610    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17611        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17612        assert_eq!(m.title, "feat: x");
17613        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17614        assert!(!m.body.contains("Diff stat"), "{}", m.body);
17615        assert!(!m.body.contains("no task statement"), "{}", m.body);
17616    }
17617
17618    #[test]
17619    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17620        let mut state = review_state(&["日本語の件名"]);
17621        state.candidates[0].branch = "fix/macos-login-path".to_owned();
17622        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17623        state.candidates[0].branch = "機能/ログイン".to_owned();
17624        assert!(
17625            pr_message(&state, 'A')
17626                .title
17627                .starts_with("chore: land candidate A")
17628        );
17629    }
17630
17631    #[test]
17632    fn pr_message_review_fixes_survive_a_clean_final_round() {
17633        let mut state = review_state(&["feat: x"]);
17634        state.reviews = vec![
17635            round_with_notes(1, Some("handled the PATH case")),
17636            round_with_notes(2, None),
17637        ];
17638        let body = pr_message(&state, 'A').body;
17639        assert!(
17640            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17641            "{body}"
17642        );
17643        assert!(body.contains("### Round 1"), "{body}");
17644
17645        state.reviews = vec![
17646            round_with_notes(1, Some("first fix")),
17647            round_with_notes(2, Some("")),
17648            round_with_notes(3, Some("second fix")),
17649            round_with_notes(4, None),
17650        ];
17651        let body = pr_message(&state, 'A').body;
17652        assert!(
17653            body.contains("### Round 1") && !body.contains("first fix"),
17654            "{body}"
17655        );
17656        assert!(
17657            body.contains("### Round 3") && !body.contains("second fix"),
17658            "{body}"
17659        );
17660        assert!(body.contains("### Round 2"), "{body}");
17661    }
17662
17663    const FINAL: &str = "abcdef1234567890";
17664
17665    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17666        let mut r = round_with_notes(round, None);
17667        r.verified_head = verified.map(str::to_owned);
17668        r
17669    }
17670
17671    fn ok_gate(state: &mut RunState, head: Option<&str>) {
17672        state.gate = vec![CommandOutcome {
17673            command: "gate".to_owned(),
17674            code: Some(0),
17675            output_tail: String::new(),
17676            duration_ms: 0,
17677            resource_blocked: false,
17678        }];
17679        state.gate_ran = true;
17680        state.gate_head = head.map(str::to_owned);
17681    }
17682
17683    fn body_at(state: &RunState) -> String {
17684        pr_message_raw(
17685            state,
17686            'A',
17687            None,
17688            crate::pr_outcome::FinalHead::At(FINAL),
17689            ChangeNote::NotAsked,
17690        )
17691        .body
17692    }
17693
17694    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17695
17696    #[test]
17697    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17698        let mut state = review_state(&["feat: x"]);
17699        state.reviews = vec![
17700            round_with_notes(1, Some(SANDBOXED)),
17701            passing_round(2, Some(FINAL)),
17702        ];
17703        ok_gate(&mut state, Some(FINAL));
17704        let body = body_at(&state);
17705        assert!(!body.contains("Handled the PATH case."), "{body}");
17706        assert!(!body.contains("could not commit"), "{body}");
17707        assert!(!body.contains("could not run"), "{body}");
17708        assert!(
17709            body.contains("The fix is committed on the branch"),
17710            "{body}"
17711        );
17712        assert!(!body.contains("sandbox"), "{body}");
17713        assert!(
17714            body.contains("Tests (`verify.e2e`): passed on the final commit"),
17715            "{body}"
17716        );
17717        assert!(
17718            body.contains("Gate (`verify.gate`): passed on the final commit"),
17719            "{body}"
17720        );
17721        // The raw report is untouched in the record.
17722        assert!(
17723            state.reviews[0]
17724                .fix
17725                .as_ref()
17726                .unwrap()
17727                .notes
17728                .contains("could not commit")
17729        );
17730    }
17731
17732    #[test]
17733    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17734        let mut state = review_state(&["feat: x"]);
17735        let mut r1 = round_with_notes(1, Some(SANDBOXED));
17736        r1.fix.as_mut().unwrap().committed = false;
17737        let mut r2 = passing_round(2, Some(FINAL));
17738        r2.e2e[0].code = Some(1);
17739        state.reviews = vec![r1, r2];
17740        let body = body_at(&state);
17741        assert!(!body.contains("could not commit"), "{body}");
17742        assert!(!body.contains("could not run"), "{body}");
17743        assert!(body.contains("This round produced no commit"), "{body}");
17744        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17745        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17746        assert!(!body.contains("passed"), "{body}");
17747    }
17748
17749    #[test]
17750    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17751        let mut state = review_state(&["feat: x"]);
17752        state.reviews = vec![
17753            round_with_notes(1, Some(SANDBOXED)),
17754            passing_round(2, Some("0123456789abcdef")),
17755        ];
17756        ok_gate(&mut state, Some("0123456789abcdef"));
17757        let body = body_at(&state);
17758        assert!(!body.contains("HTTP tests could not run"), "{body}");
17759        // The commit claim is still settled by the structured record.
17760        assert!(!body.contains("could not commit"), "{body}");
17761        assert!(
17762            body.contains("passed on an earlier commit (`0123456`)"),
17763            "{body}"
17764        );
17765        assert!(!body.contains("passed on the final commit"), "{body}");
17766
17767        // A gate recorded before `gate_head` existed is "not recorded".
17768        ok_gate(&mut state, None);
17769        let body = body_at(&state);
17770        assert!(
17771            body.contains("passed on a commit that was not recorded"),
17772            "{body}"
17773        );
17774    }
17775
17776    #[test]
17777    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17778        let state = state_with_summary(
17779            "add retries to the client",
17780            "TITLE: feat: retries\n\nDid it.",
17781        );
17782        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17783        assert_eq!(m.title, "feat: retries");
17784        assert!(
17785            m.body.contains("<summary>Original task</summary>"),
17786            "{}",
17787            m.body
17788        );
17789        assert!(!m.body.contains("Commits under review"), "{}", m.body);
17790        assert!(
17791            m.body
17792                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17793            "{}",
17794            m.body
17795        );
17796    }
17797
17798    #[test]
17799    fn pr_message_review_bounds_a_long_english_subject() {
17800        let long = format!("feat: {}", "word ".repeat(100));
17801        let m = pr_message(&review_state(&[&long]), 'A');
17802        assert!(m.title.starts_with("feat: word"), "{}", m.title);
17803        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17804    }
17805
17806    #[test]
17807    fn pr_message_implementation_run_is_unchanged_by_review_support() {
17808        let state = state_with_summary("add retries\n\ndetails", "- did some things");
17809        let m = pr_message(&state, 'A');
17810        assert_eq!(m.title, "add retries");
17811        assert!(m.body.contains("<summary>Original task</summary>"));
17812        assert!(!m.body.contains("Commits under review"));
17813        assert_eq!(landing_subject_source(&state), state.instruction);
17814    }
17815
17816    #[test]
17817    fn review_run_squash_subject_is_the_change_not_the_prompt() {
17818        let state = review_state(&["feat: the change", "fix: typo"]);
17819        let source = landing_subject_source(&state);
17820        assert_eq!(land::merge_subject("", &source), "feat: the change");
17821        assert_eq!(
17822            land::merge_subject("magi: candidate A (uncommitted work)", &source),
17823            "feat: the change"
17824        );
17825        // An operator's rename still wins.
17826        assert_eq!(
17827            land::merge_subject("feat: renamed by hand", &source),
17828            "feat: renamed by hand"
17829        );
17830        let blank = review_state(&["日本語"]);
17831        assert!(
17832            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17833        );
17834    }
17835
17836    #[test]
17837    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17838        let state = review_state(&["feat: the change"]);
17839        let source = landing_subject_source(&state);
17840        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17841        assert_eq!(
17842            land::merge_subject(landing_title(&state, old), &source),
17843            "feat: the change"
17844        );
17845        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17846        let task = state_with_summary("add retries", "");
17847        assert_eq!(landing_title(&task, old), old);
17848    }
17849
17850    #[test]
17851    fn pr_message_describes_the_change_not_the_task() {
17852        let state = state_with_summary(
17853            "今回やってほしいこと: results projector を直す",
17854            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17855        );
17856        let m = pr_message(&state, 'A');
17857        assert_eq!(m.title, "fix(web): batch the runs list reads");
17858        assert!(
17859            m.body.starts_with("## Summary\n\n- reads run.json once"),
17860            "{}",
17861            m.body
17862        );
17863        assert!(!m.body.contains("TITLE:"), "{}", m.body);
17864        let task_at = m.body.find("今回やってほしいこと").unwrap();
17865        let details_at = m.body.find("<details>").unwrap();
17866        assert!(
17867            details_at < task_at,
17868            "the task lives inside <details>: {}",
17869            m.body
17870        );
17871        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17872        assert!(m.body.contains("magi:candidate-a"));
17873    }
17874
17875    #[test]
17876    fn pr_message_falls_back_to_the_task_without_a_title_line() {
17877        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17878        let m = pr_message(&state, 'A');
17879        assert_eq!(m.title, "add retries");
17880        assert!(
17881            m.body.contains("## Summary\n\n- did some things"),
17882            "{}",
17883            m.body
17884        );
17885
17886        let none = RunState::new(
17887            PathBuf::from("/repo"),
17888            "main".to_owned(),
17889            "abc1234".to_owned(),
17890            "add retries".to_owned(),
17891            Config::default(),
17892        );
17893        let m = pr_message(&none, 'A');
17894        assert_eq!(m.title, "add retries");
17895        assert!(!m.body.contains("## Summary"), "{}", m.body);
17896    }
17897
17898    #[test]
17899    fn pr_message_refuses_the_candidate_commit_subject() {
17900        for bad in [
17901            "TITLE: magi: candidate A (uncommitted work)",
17902            "TITLE: chore: stuff (uncommitted work)",
17903            "TITLE:   ",
17904        ] {
17905            let state = state_with_summary("add retries", bad);
17906            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17907        }
17908    }
17909
17910    #[test]
17911    fn pr_message_bounds_a_very_long_task_and_title() {
17912        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17913        let state = state_with_summary(&long, "- nothing");
17914        let m = pr_message(&state, 'A');
17915        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17916        assert!(!m.title.contains('\n'));
17917
17918        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17919        let m = pr_message(&state, 'A');
17920        assert!(m.title.starts_with("feat: "));
17921        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17922        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17923    }
17924
17925    fn long_title_of(instruction: &str) -> String {
17926        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17927    }
17928
17929    #[test]
17930    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17931        let first = "Make the landing path keep a readable title for long tasks";
17932        let line = format!(
17933            "{first}. {}",
17934            "And then keep going with more words ".repeat(20)
17935        );
17936        let t = long_title_of(&line);
17937        assert_eq!(t, first);
17938        assert!(!t.starts_with("chore: land"));
17939    }
17940
17941    #[test]
17942    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17943        let line = "word ".repeat(200);
17944        let t = long_title_of(&line);
17945        assert!(t.ends_with("word..."), "{t}");
17946        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17947    }
17948
17949    #[test]
17950    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17951        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17952            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17953        }
17954    }
17955
17956    #[test]
17957    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17958        let mut state = state_with_summary("再試行を追加する", "- nothing");
17959        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17960        state.config.graph.github_language = "ja".into();
17961        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17962    }
17963
17964    #[test]
17965    fn pr_message_title_limit_is_exact() {
17966        let at = "a".repeat(PR_TITLE_MAX);
17967        assert_eq!(long_title_of(&at), at);
17968        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17969        assert!(over.ends_with("..."), "{over}");
17970        assert_eq!(over.chars().count(), PR_TITLE_MAX);
17971    }
17972
17973    #[test]
17974    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17975        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17976        let t = long_title_of(&line);
17977        assert!(t.ends_with("..."), "{t}");
17978        assert!(t.is_ascii(), "{t}");
17979    }
17980
17981    #[test]
17982    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17983        let line = format!(
17984            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17985            "plus more filler words ".repeat(20)
17986        );
17987        let t = long_title_of(&line);
17988        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17989    }
17990
17991    #[test]
17992    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17993        let line = format!("\n\n# {}", "title words ".repeat(40));
17994        let state = state_with_summary(&line, "- nothing");
17995        let m = pr_message(&state, 'A');
17996        assert!(m.title.starts_with("title words"), "{}", m.title);
17997        assert_eq!(
17998            land::merge_subject(&m.title, &landing_subject_source(&state)),
17999            m.title
18000        );
18001        // An operator's rename wins untouched.
18002        assert_eq!(
18003            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18004            "feat: renamed by hand"
18005        );
18006    }
18007
18008    #[tokio::test]
18009    async fn github_text_rewrite_is_bounded_and_falls_back() {
18010        crate::run::pin_test_home();
18011        for (reply, accepted) in [
18012            (
18013                "TITLE: fix: retries\nAdd retries for failed requests.",
18014                true,
18015            ),
18016            (
18017                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18018                false,
18019            ),
18020            (
18021                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18022                true,
18023            ),
18024            (
18025                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18026                true,
18027            ),
18028            (
18029                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18030                true,
18031            ),
18032            (
18033                "TITLE: fix: retries\n日本語の説明です。 token=secret",
18034                false,
18035            ),
18036            ("TITLE: fix: retries\n", false),
18037        ] {
18038            let dir = tempfile::tempdir().unwrap();
18039            let mut runner = runner_at(RunStatus::Gating);
18040            runner.state = state_with_summary(
18041                "add retries",
18042                "TITLE: fix: retries\n日本語の説明を書きます。",
18043            );
18044            runner.state.repo = dir.path().to_owned();
18045            runner.state.candidates[0].worktree = dir.path().to_owned();
18046            let mut author = spec("alpha");
18047            author.command = vec![
18048                "sh".into(),
18049                "-c".into(),
18050                "printf '%s' \"$REWRITE_REPLY\"".into(),
18051            ];
18052            author.env.insert("REWRITE_REPLY".into(), reply.into());
18053            runner.state.config.agents = vec![author];
18054            seed_public_summary(&mut runner);
18055            let winner = runner.state.candidates[0].clone();
18056            let first = runner
18057                .guarded_pr_message(&winner, None, true)
18058                .await
18059                .unwrap();
18060            let message = if accepted {
18061                first.expect("a passing rewrite never asks")
18062            } else {
18063                // Withheld: the run parks on a question and posts nothing yet.
18064                assert!(first.is_none());
18065                assert!(runner.state.parked);
18066                let store = ask::Questions::open();
18067                let asked: Vec<_> = store
18068                    .list()
18069                    .into_iter()
18070                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18071                    .collect();
18072                assert_eq!(asked.len(), 1);
18073                assert!(!asked[0].detail.contains("secret"));
18074                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18075                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18076                assert_eq!(
18077                    asked[0]
18078                        .choices
18079                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18080                    rewrite.is_some()
18081                );
18082                // Silence: the question lapses and the neutral text is used,
18083                // without a second rewrite call or a second question.
18084                store
18085                    .update(&asked[0].id, |q| {
18086                        q.abandon("test");
18087                        Ok(())
18088                    })
18089                    .unwrap();
18090                let again = runner
18091                    .guarded_pr_message(&winner, None, true)
18092                    .await
18093                    .unwrap()
18094                    .expect("an abandoned question falls back");
18095                let n = store
18096                    .list()
18097                    .into_iter()
18098                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18099                    .count();
18100                assert_eq!(n, 1, "asked once per rejected text");
18101                again
18102            };
18103            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18104            assert!(!message.body.contains("secret"));
18105            assert!(!message.body.contains("prefix"));
18106            assert!(!message.title.contains("secret"));
18107            assert_eq!(
18108                message.body.contains("Add retries for failed requests."),
18109                accepted
18110            );
18111            assert_eq!(
18112                runner.state.seats["github-text-rewrite"].turns, 1,
18113                "only one rewrite invocation"
18114            );
18115            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18116            if accepted {
18117                assert!(runner.state.github_text.is_none());
18118                assert!(!runner.state.parked);
18119                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18120                assert!(message.body.contains("## Final verification"));
18121                assert!(message.body.contains("Evidence: final diff"));
18122                assert!(
18123                    message
18124                        .body
18125                        .contains(&format!("magi:run/{}", runner.state.id))
18126                );
18127            }
18128            if !accepted {
18129                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18130                assert!(
18131                    message
18132                        .body
18133                        .contains(&format!("magi:run/{}", runner.state.id))
18134                );
18135            }
18136        }
18137    }
18138
18139    fn scripted(id: &str, reply: &str) -> AgentSpec {
18140        let mut a = spec(id);
18141        a.command = vec![
18142            "sh".into(),
18143            "-c".into(),
18144            "printf '%s' \"$REWRITE_REPLY\"".into(),
18145        ];
18146        a.env.insert("REWRITE_REPLY".into(), reply.into());
18147        a
18148    }
18149
18150    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18151    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18152
18153    /// Gate tests inject generated public output, not internal reports.
18154    fn seed_public_summary(runner: &mut Runner) {
18155        let repo = &runner.state.repo;
18156        let g = |args: &[&str]| {
18157            let out = std::process::Command::new("git")
18158                .args(args)
18159                .current_dir(repo)
18160                .quiet()
18161                .output()
18162                .unwrap();
18163            assert!(out.status.success(), "git {args:?}");
18164            String::from_utf8_lossy(&out.stdout).trim().to_owned()
18165        };
18166        g(&["init", "-q", "-b", "main"]);
18167        g(&[
18168            "-c",
18169            "user.name=Test",
18170            "-c",
18171            "user.email=test@example.invalid",
18172            "commit",
18173            "-q",
18174            "--allow-empty",
18175            "-m",
18176            "feat: fixture",
18177        ]);
18178        let head = g(&["rev-parse", "HEAD"]);
18179        g(&["update-ref", "refs/remotes/origin/main", &head]);
18180        let evidence = crate::pr_outcome::summary_evidence(
18181            &runner.state,
18182            crate::pr_outcome::FinalHead::At(&head),
18183        );
18184        let generated = &runner.state.candidates[0].summary;
18185        runner.state.change_summary = Some(crate::run::ChangeSummary {
18186            base: head.clone(),
18187            head,
18188            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18189            title: summary_title(generated).unwrap(),
18190            body: summary_without_title(generated),
18191        });
18192    }
18193
18194    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18195        let mut runner = runner_at(RunStatus::Gating);
18196        runner.state = state_with_summary("add retries", JA_SUMMARY);
18197        if review_only {
18198            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18199        }
18200        runner.state.repo = dir.to_owned();
18201        runner.state.candidates[0].worktree = dir.to_owned();
18202        runner.state.config.agents = agents;
18203        seed_public_summary(&mut runner);
18204        runner
18205    }
18206
18207    fn gate_questions(runner: &Runner) -> usize {
18208        ask::Questions::open()
18209            .list()
18210            .into_iter()
18211            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18212            .count()
18213    }
18214
18215    #[tokio::test]
18216    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18217        crate::run::pin_test_home();
18218        let dir = tempfile::tempdir().unwrap();
18219        let mut runner = rewrite_runner(
18220            dir.path(),
18221            true,
18222            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18223        );
18224        runner.state.config.roles.language_judge =
18225            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18226        let winner = runner.state.candidates[0].clone();
18227        let msg = runner
18228            .guarded_pr_message(&winner, None, true)
18229            .await
18230            .unwrap()
18231            .expect("an English rewrite posts without asking");
18232        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18233        assert!(msg.body.contains("<details>kept</details>"));
18234        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18235        assert_eq!(gate_questions(&runner), 0);
18236        assert!(runner.state.github_text.is_none());
18237        assert!(!runner.state.seats.contains_key("impl-A"));
18238        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18239    }
18240
18241    #[tokio::test]
18242    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18243        crate::run::pin_test_home();
18244        let dir = tempfile::tempdir().unwrap();
18245        let mut runner = rewrite_runner(
18246            dir.path(),
18247            false,
18248            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18249        );
18250        runner.state.config.roles.language_judge =
18251            Some(crate::config::AgentChoice::One("beta".into()));
18252        runner.state.seats.insert(
18253            "impl-A".into(),
18254            SeatState::new("impl-A", "alpha", runner.state.seed),
18255        );
18256        let winner = runner.state.candidates[0].clone();
18257        let msg = runner
18258            .guarded_pr_message(&winner, None, true)
18259            .await
18260            .unwrap()
18261            .expect("rewritten");
18262        assert!(msg.body.contains("<details>kept</details>"));
18263        assert_eq!(runner.state.seats["impl-A"].turns, 0);
18264        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18265        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18266    }
18267
18268    #[tokio::test]
18269    async fn rewrite_reuses_a_resumable_implementer_session() {
18270        crate::run::pin_test_home();
18271        let dir = tempfile::tempdir().unwrap();
18272        let mut runner = rewrite_runner(
18273            dir.path(),
18274            false,
18275            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18276        );
18277        runner.state.config.roles.language_judge =
18278            Some(crate::config::AgentChoice::One("beta".into()));
18279        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18280        seat.turns = 1;
18281        runner.state.seats.insert("impl-A".into(), seat);
18282        let winner = runner.state.candidates[0].clone();
18283        runner
18284            .guarded_pr_message(&winner, None, true)
18285            .await
18286            .unwrap()
18287            .expect("rewritten");
18288        assert_eq!(runner.state.seats["impl-A"].turns, 2);
18289        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18290    }
18291
18292    #[tokio::test]
18293    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18294        crate::run::pin_test_home();
18295        let dir = tempfile::tempdir().unwrap();
18296        let mut broken = spec("bad");
18297        broken.command = vec!["false".into()];
18298        let mut runner = rewrite_runner(
18299            dir.path(),
18300            true,
18301            vec![
18302                broken,
18303                scripted("noformat", "Add retries."),
18304                scripted("good", EN_REWRITE),
18305            ],
18306        );
18307        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18308            "bad".into(),
18309            "noformat".into(),
18310            "good".into(),
18311        ]));
18312        let winner = runner.state.candidates[0].clone();
18313        let msg = runner
18314            .guarded_pr_message(&winner, None, true)
18315            .await
18316            .unwrap()
18317            .expect("the third agent rewrites");
18318        assert!(msg.body.contains("`src/net.rs`"));
18319        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18320        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18321    }
18322
18323    #[tokio::test]
18324    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18325        crate::run::pin_test_home();
18326        // (agents, role): still Japanese; no TITLE; nothing resolvable.
18327        for (agents, role, saved) in [
18328            (
18329                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18330                Some("beta"),
18331                true,
18332            ),
18333            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18334            (Vec::new(), None, false),
18335        ] {
18336            let dir = tempfile::tempdir().unwrap();
18337            let mut runner = rewrite_runner(dir.path(), true, agents);
18338            runner.state.config.roles.language_judge =
18339                role.map(|r| crate::config::AgentChoice::One(r.into()));
18340            let winner = runner.state.candidates[0].clone();
18341            let first = runner
18342                .guarded_pr_message(&winner, None, true)
18343                .await
18344                .unwrap();
18345            assert!(first.is_none(), "withheld text parks the run");
18346            assert!(runner.state.parked);
18347            assert_eq!(gate_questions(&runner), 1);
18348            let g = runner.state.github_text.as_ref().unwrap();
18349            assert_eq!(g.rewrite.is_some(), saved);
18350            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18351        }
18352    }
18353
18354    #[tokio::test]
18355    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18356        crate::run::pin_test_home();
18357        let dir = tempfile::tempdir().unwrap();
18358        let mut runner = runner_at(RunStatus::Gating);
18359        runner.state =
18360            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18361        runner.state.repo = dir.path().to_owned();
18362        runner.state.candidates[0].worktree = dir.path().to_owned();
18363        let mut author = spec("alpha");
18364        author.command = vec![
18365            "sh".into(),
18366            "-c".into(),
18367            "printf '%s' \"$REWRITE_REPLY\"".into(),
18368        ];
18369        author.env.insert(
18370            "REWRITE_REPLY".into(),
18371            format!(
18372                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18373                "English sentence. ".repeat(600)
18374            ),
18375        );
18376        runner.state.config.agents = vec![author];
18377        seed_public_summary(&mut runner);
18378        let winner = runner.state.candidates[0].clone();
18379        assert!(
18380            runner
18381                .guarded_pr_message(&winner, None, true)
18382                .await
18383                .unwrap()
18384                .is_none()
18385        );
18386        let record = runner.state.github_text.clone().unwrap();
18387        let rewrite = record.rewrite.clone().unwrap();
18388        assert_eq!(rewrite.failed, ["body-language"]);
18389        assert!(
18390            rewrite
18391                .body
18392                .ends_with(&format!("magi:run/{}", runner.state.id))
18393        );
18394        let store = ask::Questions::open();
18395        let q = store.get(record.question.as_ref().unwrap()).unwrap();
18396        assert!(
18397            q.choices
18398                .contains(&crate::github_text::USE_REWRITE.to_owned())
18399        );
18400        for label in [
18401            "ORIGINAL",
18402            "FALLBACK",
18403            "REWRITE",
18404            "Rules still failed: body-language",
18405            "truncated",
18406        ] {
18407            assert!(q.detail.contains(label), "{label}");
18408        }
18409        let artifact = std::fs::read_to_string(crate::run::artifact_path(
18410            &runner.state,
18411            &crate::github_text::artifact_name(&record.fingerprint),
18412        ))
18413        .unwrap();
18414        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18415        // Restart from serialized state, then retry after a bad owner title.
18416        runner.state =
18417            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18418        store
18419            .update(&q.id, |q| {
18420                q.say("まだ日本語")?;
18421                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18422            })
18423            .unwrap();
18424        assert!(
18425            runner
18426                .guarded_pr_message(&winner, None, true)
18427                .await
18428                .unwrap()
18429                .is_none()
18430        );
18431        let retry = runner.state.github_text.as_ref().unwrap();
18432        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18433        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18434        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18435        assert!(
18436            q.posting_comparison
18437                .as_ref()
18438                .unwrap()
18439                .matches_detail(&q.detail)
18440        );
18441        store
18442            .update(&q.id, |q| {
18443                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18444            })
18445            .unwrap();
18446        let message = runner
18447            .guarded_pr_message(&winner, None, true)
18448            .await
18449            .unwrap()
18450            .unwrap();
18451        assert_eq!(message.title, rewrite.title);
18452        assert_eq!(message.body, rewrite.body);
18453        assert_eq!(
18454            runner.state.github_text.as_ref().unwrap().choice,
18455            Some(GithubTextChoice::Rewrite)
18456        );
18457        let again = runner
18458            .guarded_pr_message(&winner, None, true)
18459            .await
18460            .unwrap()
18461            .unwrap();
18462        assert_eq!(again.title, message.title);
18463        assert_eq!(again.body, message.body);
18464        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18465        assert!(
18466            !runner
18467                .state
18468                .events
18469                .iter()
18470                .any(|e| e.message.contains(&rewrite.title)
18471                    || e.message.contains("English sentence."))
18472        );
18473    }
18474
18475    #[tokio::test]
18476    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18477        crate::run::pin_test_home();
18478        let dir = tempfile::tempdir().unwrap();
18479        let mut runner = runner_at(RunStatus::Gating);
18480        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18481        runner.state.repo = dir.path().to_owned();
18482        runner.state.candidates[0].worktree = dir.path().to_owned();
18483        seed_public_summary(&mut runner);
18484        let winner = runner.state.candidates[0].clone();
18485        assert!(
18486            runner
18487                .guarded_pr_message(&winner, None, true)
18488                .await
18489                .unwrap()
18490                .is_none()
18491        );
18492        let record = runner.state.github_text.as_ref().unwrap();
18493        assert!(record.rewrite.is_none());
18494        // Records written before the additive field deserialize with no rewrite.
18495        let mut old = serde_json::to_value(record).unwrap();
18496        old.as_object_mut().unwrap().remove("rewrite");
18497        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18498        assert!(old.rewrite.is_none());
18499        let id = record.question.clone().unwrap();
18500        ask::Questions::open()
18501            .update(&id, |q| {
18502                q.choices.push(crate::github_text::USE_REWRITE.into());
18503                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18504            })
18505            .unwrap();
18506        let message = runner
18507            .guarded_pr_message(&winner, None, true)
18508            .await
18509            .unwrap()
18510            .unwrap();
18511        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18512        assert_eq!(
18513            runner.state.github_text.as_ref().unwrap().choice,
18514            Some(GithubTextChoice::Fallback)
18515        );
18516    }
18517
18518    #[tokio::test]
18519    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18520        crate::run::pin_test_home();
18521        let dir = tempfile::tempdir().unwrap();
18522        let mut runner = runner_at(RunStatus::Gating);
18523        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18524        runner.state.repo = dir.path().to_owned();
18525        runner.state.candidates[0].worktree = dir.path().to_owned();
18526        seed_public_summary(&mut runner);
18527        let winner = runner.state.candidates[0].clone();
18528        assert!(
18529            runner
18530                .guarded_pr_message(&winner, None, true)
18531                .await
18532                .unwrap()
18533                .is_none()
18534        );
18535        let store = ask::Questions::open();
18536        let find = |run: &str| {
18537            store
18538                .list()
18539                .into_iter()
18540                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18541                .unwrap()
18542        };
18543        let run = runner.state.id.clone();
18544        let q = find(&run);
18545        let fp = runner
18546            .state
18547            .github_text
18548            .as_ref()
18549            .unwrap()
18550            .fingerprint
18551            .clone();
18552        let name = crate::github_text::artifact_name(&fp);
18553        let path = crate::run::artifact_path(&runner.state, &name);
18554        assert!(q.detail.contains(&path.display().to_string()));
18555        assert!(q.detail.contains("title-language: title"));
18556        let saved = std::fs::read_to_string(&path).expect("artifact written");
18557        assert!(saved.contains("日本語のタイトル"));
18558        assert!(q.cwd.is_none());
18559        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18560        // A bad replacement is refused and asked about once more.
18561        let say = |id: &str, text: &str, choice: &str| {
18562            store
18563                .update(id, |q| {
18564                    q.thread.push(ask::Turn {
18565                        who: ask::Who::Operator,
18566                        body: text.to_owned(),
18567                        at: jiff::Timestamp::now(),
18568                        note: None,
18569                    });
18570                    q.answer(ask::Answer::Choice(choice.to_owned()))
18571                })
18572                .unwrap();
18573        };
18574        say(&q.id, "まだ日本語", "use my text");
18575        assert!(
18576            runner
18577                .guarded_pr_message(&winner, None, true)
18578                .await
18579                .unwrap()
18580                .is_none()
18581        );
18582        let second = find(&run);
18583        assert_ne!(second.id, q.id);
18584        assert!(second.detail.contains("did not pass"));
18585        // The retry still compares the original text; the rejected
18586        // replacement is named by category only.
18587        assert!(second.detail.contains("日本語のタイトル"));
18588        assert!(!second.detail.contains("まだ日本語"));
18589        assert!(second.detail.contains(&path.display().to_string()));
18590        say(&second.id, "fix: retry failed requests", "use my text");
18591        let message = runner
18592            .guarded_pr_message(&winner, None, true)
18593            .await
18594            .unwrap()
18595            .expect("a vetted replacement is posted");
18596        assert_eq!(message.title, "fix: retry failed requests");
18597        assert_eq!(
18598            runner
18599                .state
18600                .github_text
18601                .as_ref()
18602                .unwrap()
18603                .chosen_title
18604                .as_deref(),
18605            Some("fix: retry failed requests")
18606        );
18607        assert!(runner.state.github_text.as_ref().unwrap().resolved);
18608        // Resumed again: the saved decision is applied, nothing is re-asked.
18609        let again = runner
18610            .guarded_pr_message(&winner, None, true)
18611            .await
18612            .unwrap()
18613            .unwrap();
18614        assert_eq!(again.title, message.title);
18615        assert_eq!(
18616            store
18617                .list()
18618                .into_iter()
18619                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18620                .count(),
18621            2
18622        );
18623    }
18624
18625    #[test]
18626    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18627        // What magi itself writes stays English under any configured language,
18628        // so a future localisation of these headings fails here. (The agents'
18629        // own text is also checked by the posting gate.)
18630        let mut state = state_with_summary(
18631            "add retries",
18632            "TITLE: fix(web): batch reads\n- reads run.json once",
18633        );
18634        state.config.graph.language = "ja".to_owned();
18635        let m = pr_message(&state, 'A');
18636        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18637        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18638
18639        // The task is the operator's own text: it goes in untouched, and the
18640        // fallback title (no summary) may be in its language too.
18641        let task = "今回やってほしいこと: results projector を直す";
18642        let mut state = state_with_summary(task, "- no title line");
18643        state.config.graph.language = "ja".to_owned();
18644        let m = pr_message(&state, 'A');
18645        assert_eq!(
18646            m.title,
18647            format!("chore: land candidate A of run {}", state.id)
18648        );
18649        assert!(
18650            m.body.contains(&format!(
18651                "<summary>Original task</summary>\n\n{task}\n\n</details>"
18652            )),
18653            "{}",
18654            m.body
18655        );
18656    }
18657
18658    #[test]
18659    fn pr_message_scrubs_home_paths_and_addresses() {
18660        let state = state_with_summary(
18661            "fix it in /Users/someone/src/x",
18662            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18663        );
18664        let m = pr_message(&state, 'A');
18665        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18666            assert!(!m.body.contains(leak), "{}", m.body);
18667        }
18668        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18669    }
18670
18671    #[test]
18672    fn pr_message_survives_a_task_that_closes_details() {
18673        let state = state_with_summary("a </details> b", "TITLE: fix: x");
18674        let m = pr_message(&state, 'A');
18675        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18676    }
18677
18678    #[test]
18679    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18680        let cmd = manual_merge_command(
18681            MergeStyle::Squash,
18682            Path::new("/repo"),
18683            "b",
18684            "fix: \"quoted\" $(x) `y`\n\nbody",
18685        );
18686        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18687    }
18688
18689    #[test]
18690    fn manual_merge_command_matches_the_configured_style() {
18691        let repo = Path::new("/repo");
18692        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18693
18694        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18695        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18696
18697        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18698        assert_eq!(
18699            squash,
18700            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18701             \"Merge magi run 0832 (candidate A)\""
18702        );
18703
18704        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18705        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18706    }
18707
18708    #[test]
18709    fn a_nudge_gets_a_quarter_of_the_budget() {
18710        // The judge and implement budgets magi ships with.
18711        assert_eq!(retry_budget(secs(1200), true), secs(300));
18712        assert_eq!(retry_budget(secs(3600), true), secs(900));
18713    }
18714
18715    #[test]
18716    fn a_resent_prompt_keeps_the_whole_budget() {
18717        // The seat kept no context, so the retry is the original job again and
18718        // shortening it would only guarantee a second failure.
18719        assert_eq!(retry_budget(secs(1200), false), secs(1200));
18720        assert_eq!(retry_budget(secs(60), false), secs(60));
18721    }
18722
18723    #[test]
18724    fn the_floor_never_exceeds_the_original_budget() {
18725        // A short configured timeout must not be *raised* by the floor: the
18726        // operator asked for a bound, and a retry may not outlast the attempt
18727        // it is retrying.
18728        assert_eq!(retry_budget(secs(60), true), secs(60));
18729        assert_eq!(retry_budget(secs(480), true), secs(120));
18730        assert_eq!(retry_budget(secs(0), true), secs(0));
18731    }
18732
18733    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18734        agent::CommandEvidence {
18735            id: "item1".to_owned(),
18736            description: "cargo test".to_owned(),
18737            exit_code,
18738            result_summary: String::new(),
18739            source: "codex".to_owned(),
18740        }
18741    }
18742
18743    #[test]
18744    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18745        // No evidence is not the same fact as unconfirmed evidence: a
18746        // backend with no adapter, or a reply that ran no commands at all,
18747        // must not be misread as carrying a dangling job.
18748        assert!(!has_unconfirmed_command(&[]));
18749    }
18750
18751    #[test]
18752    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18753        // Deliberately not a check on the exit code's *value*: a fixer
18754        // legitimately runs something that fails mid-iteration before it
18755        // succeeds, and that must never by itself reopen a valid report.
18756        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18757        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18758        assert!(!has_unconfirmed_command(&[
18759            evidence(Some(0)),
18760            evidence(Some(101))
18761        ]));
18762    }
18763
18764    #[test]
18765    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18766        assert!(has_unconfirmed_command(&[
18767            evidence(Some(0)),
18768            evidence(None)
18769        ]));
18770    }
18771
18772    #[test]
18773    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18774        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18775        assert_eq!(
18776            verified_noop_claim(true, &[], text).as_deref(),
18777            Some("already fixed by b32cfc4, on main.")
18778        );
18779    }
18780
18781    #[test]
18782    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18783        // A timeout or a bad exit code reads as the ordinary loss it is,
18784        // whatever the reply's own prose claims.
18785        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18786        assert!(verified_noop_claim(false, &[], text).is_none());
18787    }
18788
18789    #[test]
18790    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18791        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18792        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18793        // A confirmed command alongside the marker is fine.
18794        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18795    }
18796
18797    #[test]
18798    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18799        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18800    }
18801
18802    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
18803    /// pair, in order, labelled A, B, C, ...
18804    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18805        runner.state.candidates = shape
18806            .iter()
18807            .enumerate()
18808            .map(|(i, &(empty, verified))| Candidate {
18809                index: i,
18810                label: (b'A' + i as u8) as char,
18811                agent: "sonnet".to_owned(),
18812                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18813                worktree: PathBuf::from(format!("/wt/{i}")),
18814                summary: String::new(),
18815                stat: String::new(),
18816                files: 0,
18817                commits: 0,
18818                empty,
18819                failed: None,
18820                verified_noop: verified.map(str::to_owned),
18821                duration_ms: 0,
18822                folded: false,
18823            })
18824            .collect();
18825    }
18826
18827    #[test]
18828    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18829        ask_test_home();
18830        let mut runner = runner_at(RunStatus::Implementing);
18831        set_candidates(
18832            &mut runner,
18833            &[
18834                (true, Some("already on main at b32cfc4")),
18835                (true, Some("same fix, see the existing test")),
18836            ],
18837        );
18838
18839        runner
18840            .after_implement()
18841            .expect("a verified no-op is not an error");
18842
18843        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18844    }
18845
18846    #[test]
18847    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18848        ask_test_home();
18849        let mut runner = runner_at(RunStatus::Implementing);
18850        // Candidate A declares a verified no-op; candidate B simply wrote
18851        // nothing and said nothing about why. One candidate's claim is not
18852        // the whole run's agreement.
18853        set_candidates(
18854            &mut runner,
18855            &[(true, Some("already on main at b32cfc4")), (true, None)],
18856        );
18857
18858        let err = runner
18859            .after_implement()
18860            .expect_err("an unverified empty candidate must still fail the run");
18861
18862        assert!(
18863            err.to_string().contains("no candidate produced a change"),
18864            "{err}"
18865        );
18866        assert_eq!(runner.state.status, RunStatus::Failed);
18867    }
18868
18869    #[test]
18870    fn after_implement_still_fails_an_ordinary_all_empty_run() {
18871        ask_test_home();
18872        let mut runner = runner_at(RunStatus::Implementing);
18873        set_candidates(&mut runner, &[(true, None), (true, None)]);
18874
18875        let err = runner
18876            .after_implement()
18877            .expect_err("no candidate declared anything; this is an ordinary failure");
18878
18879        assert!(
18880            err.to_string().contains("no candidate produced a change"),
18881            "{err}"
18882        );
18883        assert_eq!(runner.state.status, RunStatus::Failed);
18884    }
18885}
18886
18887/// A graft rollback whose restore could not be confirmed (see
18888/// [`Runner::graft`]).
18889#[derive(Debug)]
18890struct GraftRestoreFailed(String);
18891
18892impl std::fmt::Display for GraftRestoreFailed {
18893    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18894        write!(
18895            f,
18896            "could not restore the winner to its pre-graft commit: {}",
18897            self.0
18898        )
18899    }
18900}
18901
18902impl std::error::Error for GraftRestoreFailed {}
18903
18904/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
18905/// not the agent: they never draw on an agent's own timeout, and running out of
18906/// them never reads as an agent failure.
18907#[cfg(test)]
18908mod resource_wait_tests {
18909    use super::*;
18910    use std::collections::BTreeMap;
18911
18912    fn sh(id: &str, script: &str) -> AgentSpec {
18913        AgentSpec {
18914            id: id.to_owned(),
18915            kind: crate::config::AgentKind::Command,
18916            model: None,
18917            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18918            extra_args: Vec::new(),
18919            env: BTreeMap::new(),
18920            prompt_delivery: None,
18921        }
18922    }
18923
18924    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18925        SeatJob {
18926            seat: SeatState::new("impl-A", &spec.id, 1),
18927            spec,
18928            cwd: dir.to_path_buf(),
18929            prompt: "do it".to_owned(),
18930            timeout,
18931            allow_write: true,
18932            sessions: false,
18933            artifacts: dir.join("art"),
18934            stem: "impl-A".to_owned(),
18935            handover: None,
18936        }
18937    }
18938
18939    fn state(resource_wait_secs: u64) -> RunState {
18940        let mut config = Config::default();
18941        config.graph.timeout_resource_wait = resource_wait_secs;
18942        RunState::new(
18943            PathBuf::from("/repo"),
18944            "main".to_owned(),
18945            "abc1234".to_owned(),
18946            "task".to_owned(),
18947            config,
18948        )
18949    }
18950
18951    fn hold(cache: &Path) -> crate::cache::Guard {
18952        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18953        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18954            crate::cache::AcquireOutcome::Acquired(g) => g,
18955            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18956        }
18957    }
18958
18959    async fn run(
18960        jobs: Vec<SeatJob>,
18961        sem: Arc<Semaphore>,
18962        cache: Option<&Path>,
18963        st: &mut RunState,
18964    ) -> Vec<(usize, SeatState, AgentOutcome)> {
18965        let prompts = Prompts::default();
18966        let ctx = WaveCtx {
18967            run: "r",
18968            node: "implement",
18969            prompts: &prompts,
18970            cache,
18971            round: None,
18972            carry_seats: false,
18973            pauses: None,
18974        };
18975        wave(jobs, sem, &ctx, st, 0).await
18976    }
18977
18978    #[tokio::test]
18979    async fn verification_cache_wait_preserves_the_command_budget() {
18980        crate::run::pin_test_home();
18981        let tmp = tempfile::tempdir().unwrap();
18982        let cache = tmp.path().join("target");
18983        let guard = hold(&cache);
18984        let release = tokio::spawn(async move {
18985            tokio::time::sleep(Duration::from_millis(100)).await;
18986            guard.release();
18987        });
18988        let mut st = state(10);
18989        let budget = Duration::from_millis(50);
18990        let wait = repository_wait(&st, None);
18991        let (outcomes, _) = with_cache_lease(
18992            &mut st,
18993            Some(&cache),
18994            "gate",
18995            "gate",
18996            tmp.path(),
18997            "abc1234",
18998            budget,
18999            "test gate",
19000            &wait,
19001            |_, available| async move {
19002                assert_eq!(
19003                    available, budget,
19004                    "contention must not shorten verification"
19005                );
19006                (Vec::new(), false, Vec::new())
19007            },
19008        )
19009        .await
19010        .unwrap();
19011        release.await.unwrap();
19012        assert!(outcomes.is_empty(), "{outcomes:?}");
19013        assert!(
19014            st.events
19015                .iter()
19016                .any(|event| event.message.contains("waiting for"))
19017        );
19018        assert!(st.resource_wait.is_none());
19019        // The verification body no longer owns the lease after returning.
19020        hold(&cache).release();
19021    }
19022
19023    #[tokio::test]
19024    async fn verification_cache_wait_expires_without_starting_commands() {
19025        crate::run::pin_test_home();
19026        let tmp = tempfile::tempdir().unwrap();
19027        let cache = tmp.path().join("target");
19028        let _held = hold(&cache);
19029        let mut st = state(0);
19030        let wait = repository_wait(&st, None);
19031        let (outcomes, _) = with_cache_lease(
19032            &mut st,
19033            Some(&cache),
19034            "gate",
19035            "gate",
19036            tmp.path(),
19037            "abc1234",
19038            Duration::from_secs(60),
19039            "test gate",
19040            &wait,
19041            |_, _| async { panic!("commands must not start without the cache lease") },
19042        )
19043        .await
19044        .unwrap();
19045        assert_eq!(outcomes.len(), 1);
19046        assert!(outcomes[0].resource_blocked);
19047        assert!(st.resource_wait.is_none());
19048    }
19049
19050    #[tokio::test]
19051    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19052        crate::run::pin_test_home();
19053        let tmp = tempfile::tempdir().unwrap();
19054        let cache = tmp.path().join("target-a");
19055        let guard = hold(&cache);
19056        // Held for longer than the agent's whole 1s budget.
19057        tokio::spawn(async move {
19058            tokio::time::sleep(Duration::from_millis(1500)).await;
19059            guard.release();
19060        });
19061        let mut st = state(60);
19062        let results = run(
19063            vec![job(
19064                sh("alpha", "echo ok"),
19065                tmp.path(),
19066                Duration::from_secs(1),
19067            )],
19068            Arc::new(Semaphore::new(1)),
19069            Some(&cache),
19070            &mut st,
19071        )
19072        .await;
19073        assert!(
19074            matches!(results[0].2, AgentOutcome::Ok(_)),
19075            "the wait must not be charged to the agent"
19076        );
19077        assert!(st.resource_block.is_none());
19078        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19079        assert!(st.resource_wait.is_none() && st.active.is_empty());
19080    }
19081
19082    #[tokio::test]
19083    #[cfg(unix)]
19084    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19085        use std::os::fd::AsRawFd as _;
19086        crate::run::pin_test_home();
19087        let tmp = tempfile::tempdir().unwrap();
19088        crate::git::git(tmp.path(), &["init"]).await.unwrap();
19089        let common = tmp.path().join(".git");
19090        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19091        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19092        assert_eq!(
19093            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19094            0
19095        );
19096        let mut st = state(1);
19097        st.config.coordination = crate::config::test_coordination();
19098        let mut reader = job(
19099            sh("reader", "echo must-not-run"),
19100            tmp.path(),
19101            Duration::from_secs(30),
19102        );
19103        reader.allow_write = false;
19104        let results = run(
19105            vec![reader.clone()],
19106            Arc::new(Semaphore::new(1)),
19107            None,
19108            &mut st,
19109        )
19110        .await;
19111        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19112        assert_eq!(results[0].1.turns, 0);
19113        assert!(FailClass::of(&results[0].2).is_none());
19114        assert_eq!(
19115            st.resource_block.as_ref().unwrap().kind,
19116            "repository coordination"
19117        );
19118        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19119        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19120        drop(external);
19121        let editor = crate::repo_lock::SeatLease::for_checkout(
19122            tmp.path(),
19123            true,
19124            crate::config::test_coordination().as_ref(),
19125        )
19126        .await
19127        .unwrap();
19128        let pause = Pause::new();
19129        let interrupt = Pause::new();
19130        let prompts = Prompts::default();
19131        let ctx = WaveCtx {
19132            run: "r",
19133            node: "review",
19134            prompts: &prompts,
19135            cache: None,
19136            round: None,
19137            carry_seats: false,
19138            pauses: Some((&pause, &interrupt)),
19139        };
19140        let (results, ()) = tokio::join!(
19141            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19142            async {
19143                tokio::time::sleep(Duration::from_millis(50)).await;
19144                pause.park();
19145            }
19146        );
19147        assert!(
19148            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19149        );
19150        assert_eq!(results[0].1.turns, 0);
19151        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19152        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19153        drop(editor);
19154    }
19155
19156    #[tokio::test]
19157    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19158        crate::run::pin_test_home();
19159        let tmp = tempfile::tempdir().unwrap();
19160        let cache = tmp.path().join("target-b");
19161        let _held = hold(&cache);
19162        let mut st = state(1);
19163        let results = run(
19164            vec![job(
19165                sh("alpha", "echo ok"),
19166                tmp.path(),
19167                Duration::from_secs(1),
19168            )],
19169            Arc::new(Semaphore::new(1)),
19170            Some(&cache),
19171            &mut st,
19172        )
19173        .await;
19174        let (_, seat, out) = &results[0];
19175        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19176        assert!(wave_blocked(&results));
19177        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19178        assert_eq!(seat.turns, 0, "nothing was asked");
19179        assert!(
19180            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19181            "no agent was spawned"
19182        );
19183        assert!(FailClass::of(out).is_none(), "not a failure class");
19184        let block = st.resource_block.as_ref().expect("recorded for the resume");
19185        assert_eq!(block.kind, "shared build cache");
19186        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19187        assert!(st.active.is_empty());
19188    }
19189
19190    #[tokio::test]
19191    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19192        crate::run::pin_test_home();
19193        let tmp = tempfile::tempdir().unwrap();
19194        let mut st = state(1);
19195        let results = run(
19196            vec![job(
19197                sh("alpha", "echo ok"),
19198                tmp.path(),
19199                Duration::from_secs(1),
19200            )],
19201            Arc::new(Semaphore::new(0)),
19202            None,
19203            &mut st,
19204        )
19205        .await;
19206        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19207        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19208        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19209    }
19210
19211    #[tokio::test]
19212    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19213        crate::run::pin_test_home();
19214        let tmp = tempfile::tempdir().unwrap();
19215        let mut st = state(60);
19216        let results = run(
19217            vec![job(
19218                sh("alpha", "sleep 30"),
19219                tmp.path(),
19220                Duration::from_secs(1),
19221            )],
19222            Arc::new(Semaphore::new(1)),
19223            None,
19224            &mut st,
19225        )
19226        .await;
19227        let out = &results[0].2;
19228        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19229        let class = FailClass::of(out).expect("a failure");
19230        assert_eq!(class, FailClass::Timeout);
19231        assert!(should_hand_over(None, &class), "existing handover policy");
19232        assert!(st.resource_block.is_none());
19233    }
19234}