Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54    Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58    ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62/// How much verification output is kept and fed back to the fixer.
63const OUTPUT_TAIL: usize = 8_000;
64
65/// Bytes of a failing command's output kept in an event, so the reason a run
66/// stopped is readable from the report without opening `run.json`.
67const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
70/// command's pid before releasing the build cache's lease.
71const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
74/// command's pid to actually exit before giving up and releasing anyway.
75///
76/// A timeout means the process was asked to die (`kill_on_drop`,
77/// `start_kill`), not that it already has — on Windows in particular that can
78/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
79/// the instant the command returns would let the very next acquirer (this
80/// run's own next round, another run's verification, the janitor's prune)
81/// start touching the same directory while it might still be writing to it,
82/// so this polls the actual pid — real confirmation, not a fixed guess —
83/// until it is gone or this ceiling is reached. It is still not full
84/// process-tree reaping: a grandchild the timed-out process spawned and that
85/// outlives it independently is invisible to a pid check, and continuing to
86/// observe and collect *that* stays a different piece of work with its own
87/// owner. Set generously because the common case returns early the moment
88/// the pid is confirmed gone, not because every timeout pays this in full.
89const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91/// Consecutive review rounds with no tree progress (see
92/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
93/// instead of spending the rest of the round budget.
94///
95/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
96/// legitimately finds nothing left to change (its previous round's fix already
97/// covered it, and this round's reviewers re-raised only nits) looks the same
98/// as one that is spinning, for exactly one round. Two in a row is where the
99/// two stop being distinguishable, and a review round on this workload has
100/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
101/// third attempt at a tree that has not moved twice running is pure cost.
102/// This does not touch `review_rounds` itself, which stays the operator's
103/// call.
104pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
107/// a base that moved before giving up and leaving the run `Blocked` for a
108/// person.
109///
110/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
111/// that keeps moving faster than a run can catch it is not something more
112/// rebasing fixes, it is a person's call. Not the same *number as*
113/// `land_rounds` - this budget is spent before a pull request exists, land's
114/// after - but bounded for the identical reason, so it uses the same
115/// default. Counted across both call sites in [`Runner::finish_after_tally`]
116/// (once before review, once before the gate), because either one finding
117/// the base still moving is the same signal.
118const BASE_SYNC_ROUNDS: usize = 4;
119
120/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
121/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
122/// reply held no [`FixReport`].
123///
124/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
125/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
126/// "I'll pause here until the `cargo make check` background run reports
127/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
128/// No `FixReport` was ever collected from that seat, and the run moved on to
129/// the next review round regardless.
130///
131/// Bounded independently of `review_rounds` and `graph.retries`: this
132/// recovers one seat's missing report mid-round, not a new round of review or
133/// an ordinary parse retry, and must not itself become the unbounded wait the
134/// rest of this module exists to avoid.
135const MAX_FIX_CONTINUATIONS: usize = 2;
136
137/// One queued agent invocation.
138///
139/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
140/// CLI hung up on its own stream is asked again from the same job rather than
141/// rebuilt from scratch. See [`Runner::resume_undelivered`].
142#[derive(Clone)]
143struct SeatJob {
144    spec: AgentSpec,
145    seat: SeatState,
146    cwd: PathBuf,
147    prompt: String,
148    timeout: Duration,
149    allow_write: bool,
150    sessions: bool,
151    artifacts: PathBuf,
152    stem: String,
153    /// The prompt for a seat that has been handed to another roster agent
154    /// (a fresh session): everything the original seat would have
155    /// remembered. `None` when `prompt` already carries it, as the first
156    /// ranking and the implement prompt do. Never a resume-style prompt.
157    handover: Option<String>,
158}
159
160/// How the graph reads one agent invocation.
161///
162/// Quota is split out from an ordinary failure on purpose: a rate-limited call
163/// is known to fail again if retried now, so the retry loop must not spend an
164/// attempt on it. `Dropped` is split out for the opposite reason: unlike
165/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
166/// error JSON, never the agent's answer — a caller that matched only
167/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
168/// left to read that JSON as if it were usable output. `resume_undelivered`
169/// is the only caller that acts on it; everywhere else it is reported like an
170/// ordinary failure.
171enum AgentOutcome {
172    /// A usable output.
173    Ok(AgentOutput),
174    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
175    Quota(AgentOutput),
176    /// The CLI hung up on its own stream after billed work. See
177    /// [`agent::AgentOutput::work_undelivered`].
178    Dropped(AgentOutput),
179    /// Any other failure: a timeout, a bad exit code, an empty reply.
180    Failed(String),
181    /// A resource prevented admission or interrupted execution. Not an agent
182    /// failure: never advance a fallback chain or spend an attempt on it.
183    /// Post-spawn interruptions carry a typed error into wave, which records
184    /// implementation preservation before returning this outcome. This outcome
185    /// alone is not proof that no source changes were made.
186    /// [`RunState::resource_block`] says why.
187    ResourceBlocked(String),
188}
189
190/// A request to park the run at its next node boundary.
191///
192/// Cloning is how the request travels: the loop keeps one handle and hands a
193/// clone to each [`Runner`], and every clone points at the same flag. There
194/// is no channel because there is nothing to send - the only message is
195/// "park", it is idempotent, and a flag cannot be missed by a receiver that
196/// was not listening yet.
197///
198/// The boundary is what makes this cheap. Every node writes the run's state
199/// before the next one starts, and every node skips what is already recorded:
200/// `prep` returns early once candidates exist, `implement` asks only the seats
201/// with nothing on disk, `judge` returns early once judgements exist. So a
202/// parked run resumes into exactly the node it stopped before, and no agent
203/// work is thrown away. Killing the process mid-node, by contrast, loses
204/// whatever the seats in flight had not yet written - which for an implement
205/// wave is an hour of paid work.
206///
207/// A [`Runner`] watches two independent handles of this type - see
208/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
209/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
210/// clone covering the whole daemon's lifetime and is never asked to un-park,
211/// which is correct exactly because nothing is dispatched after it fires.
212/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
213/// that parks for an interrupted task must go on to run other tasks
214/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
215/// reusing the daemon-wide one.
216#[derive(Debug, Clone, Default)]
217pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
218
219impl Pause {
220    /// A pause nobody has asked for yet.
221    #[must_use]
222    pub fn new() -> Self {
223        Self::default()
224    }
225
226    /// Ask the run to park at its next node boundary. Idempotent.
227    pub fn park(&self) {
228        self.0.store(true, Ordering::SeqCst);
229    }
230
231    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
232    /// fold into the run's own `park` event - so an operator reading the run
233    /// later knows this was a deliberate interrupt rather than a shutdown or
234    /// a binary swap. The first reason recorded wins; a park already in
235    /// flight is not relabelled by a second, unrelated request.
236    pub fn park_because(&self, reason: impl Into<String>) {
237        self.park_for(ParkCause::Interrupt {
238            reason: reason.into(),
239        });
240    }
241
242    /// [`Pause::park`], recording the structured cause. The first cause
243    /// recorded wins, like [`Pause::park_because`].
244    pub fn park_for(&self, cause: ParkCause) {
245        let mut reason_guard = self
246            .1
247            .lock()
248            .unwrap_or_else(std::sync::PoisonError::into_inner);
249        if reason_guard.is_none() {
250            *reason_guard = Some(cause);
251        }
252        drop(reason_guard);
253        self.park();
254    }
255
256    /// Has a park been asked for?
257    #[must_use]
258    pub fn parked(&self) -> bool {
259        self.0.load(Ordering::SeqCst)
260    }
261
262    /// Why the park was asked for, when the caller used [`Pause::park_because`].
263    #[must_use]
264    pub fn reason(&self) -> Option<String> {
265        self.cause().map(|c| c.label())
266    }
267
268    /// The structured cause, when one was recorded.
269    #[must_use]
270    pub fn cause(&self) -> Option<ParkCause> {
271        self.1
272            .lock()
273            .unwrap_or_else(std::sync::PoisonError::into_inner)
274            .clone()
275    }
276}
277
278/// Drives one run.
279pub struct Runner {
280    /// Run state; public so the CLI can report on it.
281    pub state: RunState,
282    roles: ResolvedRoles,
283    sem: Arc<Semaphore>,
284    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
285    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
286    /// this is never the same handle as `interrupt`.
287    pause: Pause,
288    /// Set when `magi serve`'s interrupt scheduler wants this specific run
289    /// parked at its next node boundary, to let a task marked
290    /// [`crate::queue::Task::interrupt`] run alone before this one carries
291    /// on. Unlike `pause`, a fresh, unshared handle per run - see
292    /// [`Runner::watch_interrupt`].
293    interrupt: Pause,
294}
295
296/// Where the branch's own commits start: its merge base with the base branch
297/// as the remote has it now, else with the recorded `base_commit`. A branch
298/// rebased onto a base that moved past `base_commit` would otherwise count
299/// the base's commits as its own under `base_commit..branch`.
300async fn review_base(
301    repo: &Path,
302    remote: &str,
303    base_branch: &str,
304    base_commit: &str,
305    branch: &str,
306) -> String {
307    review_base_checked(repo, remote, base_branch, base_commit, branch)
308        .await
309        .0
310}
311
312/// [`review_base`] plus whether the base was read from a freshly fetched
313/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
314/// never older than `base_commit`'s view of the base), but the answer is then
315/// not trusted to rewrite a pull request's title.
316async fn review_base_checked(
317    repo: &Path,
318    remote: &str,
319    base_branch: &str,
320    base_commit: &str,
321    branch: &str,
322) -> (String, bool) {
323    let tracking = format!("{remote}/{base_branch}");
324    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
325    if git::rev_exists(repo, &tracking).await
326        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
327        && !mb.is_empty()
328    {
329        return (mb, fresh);
330    }
331    let mb = git::merge_base(repo, base_commit, branch)
332        .await
333        .ok()
334        .filter(|mb| !mb.is_empty())
335        .unwrap_or_else(|| base_commit.to_owned());
336    (mb, false)
337}
338
339/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
340/// when git cannot say or finds nothing: a stale list is better than a wrong
341/// or empty one.
342pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
343    if !is_review_run(state) {
344        return;
345    }
346    let base = review_base(
347        &state.repo,
348        &state.config.merge.remote,
349        &state.base_branch,
350        &state.base_commit,
351        branch,
352    )
353    .await;
354    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
355        && !subjects.is_empty()
356        && state.reviewed_commits.as_ref() != Some(&subjects)
357    {
358        state.reviewed_commits = Some(subjects);
359        state.save().ok();
360    }
361}
362
363/// Subjects of the base's commits between the recorded start and the branch's
364/// merge base: what a stale `base_commit..branch` would have mistaken for the
365/// branch's own work.
366async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
367    let (base, trusted) = review_base_checked(
368        &state.repo,
369        &state.config.merge.remote,
370        &state.base_branch,
371        &state.base_commit,
372        branch,
373    )
374    .await;
375    if !trusted {
376        return None;
377    }
378    git::subjects(&state.repo, &state.base_commit, &base)
379        .await
380        .ok()
381}
382
383/// May an adopted pull request's title be replaced with `computed`? Only when
384/// it is empty, magi's own shape, or a base commit's subject that leaked in;
385/// a title a person wrote stays. Never when `computed` is itself a leak.
386pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
387    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
388    if is_leak(computed) {
389        return false;
390    }
391    let cur = current.trim();
392    cur.is_empty()
393        || cur.starts_with(REVIEW_PROMPT_OPENING)
394        || cur.starts_with("chore: land candidate")
395        || cur.starts_with("magi: candidate")
396        || is_leak(cur)
397}
398
399/// The commit a run branches from: the base branch as the remote has it.
400///
401/// Two failures this replaces. A run used to branch off `HEAD` and so refused
402/// to start on a dirty tree, which made `magi serve` decline every task for as
403/// long as the operator had work in progress - most of the time. Branching off
404/// the *local* base branch fixed that and introduced a worse one: `land` merges
405/// the winner on GitHub, nothing updates the local ref, and the next run
406/// branches off a base missing everything the previous runs landed. Two tasks
407/// in a row from a phone would have had the second silently re-implementing
408/// against stale code and opening a pull request that reverted the first.
409///
410/// Only refs move here - no checkout, no local branch, no merge - so it is safe
411/// with uncommitted work in the tree. A machine with no network still starts:
412/// the fetch may fail and the local tip is used with a warning, because
413/// refusing to run offline is a worse failure than running against a base the
414/// operator can see for themselves.
415///
416/// One function, called by both entry points. Two answers to "where does a run
417/// branch from" is the kind of drift nobody notices until a diff is wrong.
418/// Bring the local `branch` in line with `<remote>/<branch>` before a review
419/// checks it out.
420///
421/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
422/// anything other than plain `git push` from this checkout (a jj colocated
423/// workspace, another clone) moves only the remote-tracking ref - so the local
424/// one can be a stale placeholder. It moves only when local is behind the remote or is an
425/// empty placeholder that diverged from it; unpushed local work is kept, and a real
426/// divergence is refused rather than guessed at.
427async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
428    let tracking = format!("{remote}/{branch}");
429    let fetched = git::fetch(repo, remote, branch).await;
430    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
431    let local_exists = git::branch_exists(repo, branch).await?;
432    if !fresh {
433        if !local_exists {
434            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
435        }
436        tracing::warn!(
437            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
438        );
439        return Ok(());
440    }
441    let remote_sha = git::rev_parse(repo, &tracking).await?;
442    if !local_exists {
443        git::git(repo, &["branch", branch, &tracking]).await?;
444        return Ok(());
445    }
446    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
447    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
448        return Ok(());
449    }
450    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
451        // Diverged. `reconcile` settles it only when it can prove nothing is
452        // lost: a local tip that is the remote's change rebased is pushed over
453        // it (lease pinned to the tip read here), a tip whose every extra
454        // commit is empty is a placeholder the remote's work replaced, and
455        // anything else is two different changes - a question for a person.
456        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
457            .await?
458        {
459            crate::reconcile::Reconciliation::Pushed => {
460                tracing::warn!(
461                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
462                    short(&local_sha),
463                    short(&remote_sha)
464                );
465                return Ok(());
466            }
467            crate::reconcile::Reconciliation::Placeholder => {}
468            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
469        }
470    }
471    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
472    if !out.ok() {
473        bail!(
474            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
475            short(&local_sha),
476            short(&remote_sha),
477            out.stderr
478        );
479    }
480    tracing::warn!(
481        "local `{branch}` was stale: fast-forwarded {} -> {}",
482        short(&local_sha),
483        short(&remote_sha)
484    );
485    Ok(())
486}
487
488async fn resolve_base(
489    repo: &Path,
490    base_branch: &str,
491    remote: &str,
492    coordination: Option<&crate::config::Coordination>,
493) -> Result<crate::run::FetchedBase> {
494    let _lease = crate::repo_lock::SeatLease::operation_with(
495        repo,
496        coordination,
497        &crate::repo_lock::WaitPolicy::current(),
498    )
499    .await?;
500    let tracking = format!("{remote}/{base_branch}");
501    let fetched = git::fetch(repo, remote, base_branch).await;
502    if let Ok(out) = &fetched
503        && out.ok()
504        && git::rev_exists(repo, &tracking).await
505    {
506        return Ok(crate::run::FetchedBase {
507            remote: remote.to_owned(),
508            branch: base_branch.to_owned(),
509            commit: git::rev_parse(repo, &tracking).await?,
510            fetched_at: jiff::Timestamp::now(),
511        });
512    }
513    let why = match &fetched {
514        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
515        Ok(_) => format!("{remote} has no {base_branch}"),
516        Err(e) => e.to_string(),
517    };
518    // No fallback to the local branch: it may be behind, and branching off an
519    // old commit is the stale-checkout bug this check exists to prevent.
520    bail!(
521        "cannot read {tracking} ({why}); refusing to branch off the local \
522         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
523         base / remote in magi.toml. This is launcher preparation: check the \
524         launcher's Git administration access, network and repository coordination \
525         before retrying; an unattended seat cannot approve those permissions"
526    )
527}
528
529/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
530/// on an early return or a panic.
531///
532/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
533/// manual `magi fix` invocations against the same run are otherwise
534/// invisible to each other and would race to remove and recreate the same
535/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
536/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
537/// which is only ever reclaimed later, out of band, by
538/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
539/// `magi fix` invocation is not necessarily running under either of those, so
540/// nothing would ever sweep a lock a killed or crashed process left behind.
541/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
542/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
543/// lease use: an unreadable or unparsable pid, or a liveness query the
544/// platform cannot answer, reads as alive and the lock is left in place.
545struct FixClaim {
546    path: PathBuf,
547}
548
549impl FixClaim {
550    fn acquire(dir: &Path) -> Result<Self> {
551        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
552        let path = dir.join("fix.lock");
553        match Self::create(&path) {
554            Ok(claim) => Ok(claim),
555            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
556                if Self::reclaim_if_dead(&path) {
557                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
558                } else {
559                    bail!(
560                        "another repair or `magi fix` is already running for this run ({} exists)",
561                        path.display()
562                    )
563                }
564            }
565            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
566        }
567    }
568
569    fn create(path: &Path) -> std::io::Result<Self> {
570        let mut f = std::fs::OpenOptions::new()
571            .write(true)
572            .create_new(true)
573            .open(path)?;
574        use std::io::Write as _;
575        // Read back by `reclaim_if_dead` on a later, stuck invocation.
576        writeln!(f, "{}", std::process::id())?;
577        Ok(Self {
578            path: path.to_owned(),
579        })
580    }
581
582    /// True if the lock named a process confirmed dead, in which case it was
583    /// also removed. Never true on an unreadable file, an unparsable pid, or
584    /// a liveness query the platform cannot answer — see this type's own doc.
585    fn reclaim_if_dead(path: &Path) -> bool {
586        let dead = std::fs::read_to_string(path)
587            .ok()
588            .and_then(|body| body.trim().parse::<u32>().ok())
589            .is_some_and(|pid| !crate::proc::pid_alive(pid));
590        dead && std::fs::remove_file(path).is_ok()
591    }
592}
593
594impl Drop for FixClaim {
595    fn drop(&mut self) {
596        let _ = std::fs::remove_file(&self.path);
597    }
598}
599
600/// Holds the mutable runner as well as its durable lease: cancellation must
601/// clear both copies of ownership, including state a caller may save later.
602struct DriverExecution<'a> {
603    runner: &'a mut Runner,
604    driver: crate::driver::DriverGuard,
605}
606
607impl Drop for DriverExecution<'_> {
608    fn drop(&mut self) {
609        self.driver.release_local(&mut self.runner.state);
610    }
611}
612
613impl Runner {
614    /// Start a fresh run against `repo`.
615    pub async fn start(
616        repo: &Path,
617        instruction: String,
618        config: Config,
619        origin: Origin,
620    ) -> Result<Self> {
621        Self::start_naming(repo, instruction, "", config, origin).await
622    }
623
624    /// [`Runner::start`] for a queued task: `also_scan` (the task's title) is
625    /// searched for branch and commit references along with the instruction,
626    /// since a task may name the work it is about only in its title.
627    pub async fn start_naming(
628        repo: &Path,
629        instruction: String,
630        also_scan: &str,
631        config: Config,
632        origin: Origin,
633    ) -> Result<Self> {
634        let repo = git::toplevel(repo).await?;
635        let missing = agent::missing_programs(&config.agents);
636        if !missing.is_empty() {
637            bail!(
638                "these agent programs are not on PATH: {}. Fix the roster in \
639                 magi.toml or install them.",
640                missing.join(", ")
641            );
642        }
643        let base_branch =
644            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
645                .await?;
646        let fetched_base = resolve_base(
647            &repo,
648            &base_branch,
649            &config.merge.remote,
650            config.coordination.as_ref(),
651        )
652        .await?;
653        let base_commit = fetched_base.commit.clone();
654        let roles = config.resolve_roles()?;
655        let max_parallel = config.graph.max_parallel.max(1);
656        // A task that points at work already in the repository starts from
657        // it; what the repository says about each reference is recorded.
658        let seeds = refs::resolve(
659            &repo,
660            &base_commit,
661            &config.merge.remote,
662            &format!("{also_scan}\n{instruction}"),
663        )
664        .await;
665        refs::plan(&repo, &seeds).await?;
666        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
667        // Recorded before the first save, so a crash right after minting
668        // cannot leave a run with no origin. Legibility only: nothing reads it
669        // to decide anything.
670        state.fetched_base = Some(fetched_base);
671        state.origin = Some(origin);
672        for seed in &seeds {
673            state.event(
674                "seed",
675                refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
676            );
677        }
678        state.seeds = seeds;
679        state.event("start", format!("run {} created", state.id));
680        state.save()?;
681        Ok(Self {
682            state,
683            roles,
684            sem: Arc::new(Semaphore::new(max_parallel)),
685            pause: Pause::new(),
686            interrupt: Pause::new(),
687        })
688    }
689
690    /// Open a review-only run against work that already exists on `branch`.
691    ///
692    /// The expensive half of the graph is the implement wave — measured at
693    /// 111 and 134 internal tool-loop turns on this repository, against a
694    /// handful for a judge or a reviewer. The cheap half is worth running on
695    /// hand-written work too, and there was no way to reach it.
696    ///
697    /// No new state and no schema change are needed: a run with **one** viable
698    /// candidate and a tally already decided degrades `execute` to exactly
699    /// review → gate → merge, because `judge` skips a single-candidate field,
700    /// `deliberate` has fewer than two first choices to reconcile, `vote`
701    /// returns early, `tally` is already present and `fold_losers` has no
702    /// losers. Resuming such a run therefore does the right thing as well.
703    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
704        Self::review_taking_over(repo, branch, config, None, origin).await
705    }
706
707    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
708    /// at the same task still has `branch` checked out, its worktree is
709    /// released first if that is safe (see [`crate::handover`]), and the
710    /// review refuses with the reason if it is not. `None` is a hand-run
711    /// review: it has no earlier attempts, so only a worktree of a dead run
712    /// magi recorded itself can be released.
713    pub async fn review_taking_over(
714        repo: &Path,
715        branch: &str,
716        config: Config,
717        takeover: Option<crate::handover::Takeover>,
718        origin: Origin,
719    ) -> Result<Self> {
720        let repo = git::toplevel(repo).await?;
721        let missing = agent::missing_programs(&config.agents);
722        if !missing.is_empty() {
723            bail!(
724                "these agent programs are not on PATH: {}. Fix the roster in \
725                 magi.toml or install them.",
726                missing.join(", ")
727            );
728        }
729        let base_branch =
730            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
731                .await?;
732        if base_branch == branch {
733            bail!("`{branch}` is the base branch; there is nothing to review against");
734        }
735        let fetched_base = resolve_base(
736            &repo,
737            &base_branch,
738            &config.merge.remote,
739            config.coordination.as_ref(),
740        )
741        .await?;
742        let base_commit = fetched_base.commit.clone();
743
744        let roles = config.resolve_roles()?;
745        let max_parallel = config.graph.max_parallel.max(1);
746        let mut state = RunState::new(
747            repo.clone(),
748            base_branch,
749            base_commit.clone(),
750            String::new(),
751            config,
752        );
753        state.fetched_base = Some(fetched_base);
754        state.origin = Some(origin);
755
756        // Released before anything else touches the branch: a stale local
757        // branch is moved with `git branch -f`, which git refuses while an
758        // earlier attempt's worktree still has it checked out. Everything
759        // after this point that can fail puts the old run back.
760        // A hand-run review has no task, hence no earlier attempts, but a
761        // worktree of a dead run magi made may still be released.
762        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
763            earlier: Vec::new(),
764            home: crate::run::home(),
765            choice: None,
766            task_instruction: None,
767        });
768        let wait = repository_wait(&state, None);
769        let coordination_config = state.config.coordination.clone();
770        let coordination =
771            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
772                .await?;
773        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
774        if let Some(released) = &released {
775            state.event(
776                "release",
777                format!(
778                    "took `{branch}` over from run {}: its worktree was released: {}",
779                    crate::run::short_of(&released.old_id),
780                    released.audit
781                ),
782            );
783        }
784        // The owner's answer to an earlier divergence question is applied
785        // here: after the release (git will not move a checked-out branch)
786        // and before the sync that would otherwise ask again.
787        if let Some(choice) = takeover.choice.as_ref()
788            && let Err(e) =
789                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
790                    .await
791        {
792            if let Some(released) = &released {
793                released.restore(&repo, branch).await;
794            }
795            return Err(e.context("applying the owner's answer about the diverged branch"));
796        }
797        // An earlier attempt of this task that published the task head, and
798        // whose work is the branch under review, hands that head over: the
799        // review run updates the same pull request. Without one (a legacy
800        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
801        // own head and its own open pull request is adopted as before.
802        for id in takeover.earlier.iter().rev() {
803            if let Ok(old) = RunState::load_under(id, &takeover.home)
804                && let Some(head) = old.pr_head.clone()
805                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
806            {
807                state.pr_head = Some(head);
808                break;
809            }
810        }
811        drop(coordination);
812        let opened = Self::open_review(
813            &repo,
814            branch,
815            state,
816            roles,
817            max_parallel,
818            base_commit,
819            &takeover,
820        )
821        .await;
822        if opened.is_err()
823            && let Some(released) = &released
824        {
825            let _coordination = crate::repo_lock::SeatLease::operation_with(
826                &repo,
827                coordination_config.as_ref(),
828                &wait,
829            )
830            .await?;
831            released.restore(&repo, branch).await;
832        }
833        opened
834    }
835
836    /// Recover exactly the operator-selected public implementation, without touching old sources.
837    pub async fn recover_pr(
838        repo: &Path,
839        task: &str,
840        selection: crate::pr_recovery::Selection,
841        mut config: Config,
842    ) -> Result<Self> {
843        selection.configure(&mut config)?;
844        selection.check_task(task, &selection.instruction)?;
845        let base_branch =
846            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
847                .await?;
848        let fetched_base = resolve_base(
849            repo,
850            &base_branch,
851            &config.merge.remote,
852            config.coordination.as_ref(),
853        )
854        .await?;
855        let base_commit = fetched_base.commit.clone();
856        selection
857            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
858            .await?;
859        Box::pin(selection.verify_source(repo)).await?;
860        if let Some(proof) = &selection.source_proof {
861            anyhow::ensure!(
862                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
863                "source rebase base is outside the fetched landing base"
864            );
865        }
866        let roles = config.resolve_roles()?;
867        let max_parallel = config.graph.max_parallel.max(1);
868        let mut state = RunState::new(
869            repo.to_path_buf(),
870            base_branch,
871            base_commit.clone(),
872            selection.instruction.clone(),
873            config,
874        );
875        state.fetched_base = Some(fetched_base);
876        state.origin = Some(crate::run::Origin::queue(task));
877        state.pr_head = Some(selection.branch.clone());
878        state.pr_recovery = Some(selection.clone());
879        crate::pr_head::check_predecessors(&state, &selection.branch).await?;
880        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
881    }
882
883    async fn open_selected(
884        repo: &Path,
885        state: RunState,
886        roles: ResolvedRoles,
887        max_parallel: usize,
888        base_commit: String,
889        selection: crate::pr_recovery::Selection,
890    ) -> Result<Self> {
891        let mut state = state;
892        let branch = state.branch_for('A');
893        let wait = repository_wait(&state, None);
894        let coordination = coordinate_operation(&mut state, repo, &wait).await?;
895        git::git(repo, &["branch", &branch, selection.source()]).await?;
896        drop(coordination);
897        state.event(
898            "recovery",
899            format!(
900                "selected implementation {} for {} (public lease {})",
901                selection.source(),
902                selection.url,
903                selection.head
904            ),
905        );
906        let takeover = crate::handover::Takeover {
907            earlier: Vec::new(),
908            home: crate::run::home(),
909            choice: None,
910            task_instruction: Some(selection.instruction.clone()),
911        };
912        state.save()?;
913        let id = state.id.clone();
914        let opened = Self::open_review(
915            repo,
916            &branch,
917            state,
918            roles,
919            max_parallel,
920            base_commit,
921            &takeover,
922        )
923        .await;
924        match opened {
925            Ok(runner) => Ok(runner),
926            Err(e) => {
927                if let Ok(mut partial) = RunState::load(&id) {
928                    partial.status = RunStatus::Blocked;
929                    partial.driver_exited = true;
930                    partial.event(
931                        "recovery",
932                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
933                    );
934                    partial.save()?;
935                }
936                Err(crate::pr_recovery::StartupFailed {
937                    run: id,
938                    cause: format!("{e:#}"),
939                }
940                .into())
941            }
942        }
943    }
944
945    /// What the newest earlier attempt that reviewed `branch` leaves for this
946    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
947    /// no earlier attempts. Reads the earlier record only: ownership of its
948    /// worktree and branch is [`crate::handover`]'s business.
949    async fn inherit(
950        repo: &Path,
951        branch: &str,
952        base_commit: &str,
953        takeover: &crate::handover::Takeover,
954    ) -> Option<crate::recovery::RecoveryRecord> {
955        use crate::recovery::{Relation, assess};
956        let newest = takeover.earlier.last()?;
957        let mut chosen = None;
958        for id in takeover.earlier.iter().rev() {
959            if let Ok(old) = RunState::load_under(id, &takeover.home)
960                && !old.reviews.is_empty()
961                && (old.winner().is_some_and(|w| w.branch == branch)
962                    || old.pr_head.as_deref() == Some(branch))
963            {
964                chosen = Some(old);
965                break;
966            }
967        }
968        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
969            .await
970            .unwrap_or_default();
971        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
972            Some(last) if last.head == tip => Relation::Same,
973            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
974            _ => Relation::Diverged,
975        };
976        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
977        Some(assess(from, chosen.as_ref(), base_commit, relation))
978    }
979
980    /// The half of [`Runner::review_taking_over`] that can fail after an
981    /// earlier attempt's worktree was released.
982    async fn open_review(
983        repo: &Path,
984        branch: &str,
985        mut state: RunState,
986        roles: ResolvedRoles,
987        max_parallel: usize,
988        base_commit: String,
989        takeover: &crate::handover::Takeover,
990    ) -> Result<Self> {
991        let wait = repository_wait(&state, None);
992        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
993        if state.pr_recovery.is_none() {
994            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
995        }
996        // The commit subjects are the closest thing to a task statement that
997        // existing work carries, and the reviewers are told as much.
998        let start = review_base(
999            repo,
1000            &state.config.merge.remote,
1001            &state.base_branch,
1002            &base_commit,
1003            branch,
1004        )
1005        .await;
1006        let log = git::log_oneline(repo, &start, branch)
1007            .await
1008            .unwrap_or_default();
1009        let log_text = if log.trim().is_empty() {
1010            "(no commit messages)"
1011        } else {
1012            log.trim()
1013        };
1014        let mut instruction = format!(
1015            "Review the work already on branch `{branch}`. There is no task \
1016             statement: what the change claims to do is whatever its commits \
1017             say.\n\n{log_text}"
1018        );
1019        // A retry of a task whose earlier attempt left reviews behind carries
1020        // that attempt's evidence, and the task's own statement with it: the
1021        // completion criteria are the task's, not whatever the commits claim.
1022        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1023            if let Some(task) = takeover
1024                .task_instruction
1025                .as_deref()
1026                .filter(|t| !t.trim().is_empty())
1027                && rec.reviewed_head.is_some()
1028            {
1029                instruction = format!(
1030                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1031                    task.trim()
1032                );
1033                rec.criteria_inherited = true;
1034            }
1035            state.event("recovery", rec.summary());
1036            state.recovery = Some(rec);
1037        }
1038        if let Some(selected) = &state.pr_recovery {
1039            instruction = selected.instruction.clone();
1040        }
1041        state.task_instruction = takeover.task_instruction.clone();
1042        state.instruction = instruction;
1043        state.reviewed_commits = Some(
1044            git::subjects(repo, &start, branch)
1045                .await
1046                .unwrap_or_default(),
1047        );
1048
1049        // An attached worktree, so the fixer's commits land on the branch under
1050        // review rather than on a detached head nobody will look at again.
1051        let worktree = state.worktree_root().join("under-review");
1052        if let Some(parent) = worktree.parent() {
1053            tokio::fs::create_dir_all(parent).await.ok();
1054        }
1055        let path = worktree.to_string_lossy().to_string();
1056        git::git(repo, &["worktree", "add", &path, branch])
1057            .await
1058            .with_context(|| {
1059                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1060            })?;
1061
1062        let setup = match ensure_setup_config(&mut state, repo).await {
1063            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1064            Err(e) => Err(e),
1065        };
1066        if let Err(e) = setup {
1067            git::worktree_remove_clean(repo, &worktree).await.ok();
1068            return Err(e);
1069        }
1070        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1071            .await
1072            .unwrap_or(0);
1073        if commits == 0 {
1074            git::worktree_remove_clean(repo, &worktree).await.ok();
1075            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1076        }
1077        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1078            .await
1079            .map(|f| f.len())
1080            .unwrap_or(0);
1081        if files == 0
1082            && let (Ok(head_tree), Ok(base_tree)) = (
1083                git::tree_of(&worktree, "HEAD").await,
1084                git::tree_of(&worktree, &base_commit).await,
1085            )
1086            && head_tree == base_tree
1087        {
1088            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1089            git::worktree_remove_clean(repo, &worktree).await.ok();
1090            bail!(
1091                "`{branch}` at {} has a tree identical to base {}; this usually means \
1092                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1093                 against `{}/{branch}`) rather than an empty change",
1094                short(&head),
1095                short(&base_commit),
1096                state.config.merge.remote
1097            );
1098        }
1099        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1100            .await
1101            .unwrap_or_default();
1102
1103        state.candidates.push(Candidate {
1104            index: 0,
1105            label: 'A',
1106            // Not an agent id on purpose: nothing in the roster wrote this, and
1107            // the stats tables must not credit anyone with a win for it.
1108            agent: EXISTING_BRANCH.to_owned(),
1109            branch: branch.to_owned(),
1110            worktree,
1111            summary: String::new(),
1112            stat,
1113            files,
1114            commits,
1115            empty: false,
1116            failed: None,
1117            verified_noop: None,
1118            duration_ms: 0,
1119            folded: false,
1120        });
1121        state.tally = Some(Tally {
1122            first_choice: BTreeMap::from([('A', 0)]),
1123            borda: BTreeMap::new(),
1124            winner: 'A',
1125            rankings: 0,
1126            unanimous_initial: false,
1127            deliberated: false,
1128            changed_votes: 0,
1129            unanimous_final: false,
1130            tie_break: None,
1131            // No panel sat, so no quorum applies. Zero judges is the correct
1132            // number for work that never competed, and must not be reported as
1133            // a collapsed panel.
1134            judges: 0,
1135            present: 0,
1136            quorum: 0,
1137            met_quorum: true,
1138            uncontested: Some("review-only run: nothing competed".to_owned()),
1139        });
1140        state.status = RunStatus::Reviewing;
1141        state.prep_complete = Some(true);
1142        state.event(
1143            "start",
1144            format!(
1145                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1146                state.id
1147            ),
1148        );
1149        state.save()?;
1150        Ok(Self {
1151            state,
1152            roles,
1153            sem: Arc::new(Semaphore::new(max_parallel)),
1154            pause: Pause::new(),
1155            interrupt: Pause::new(),
1156        })
1157    }
1158
1159    /// Reopen an existing run.
1160    pub fn resume(id: &str) -> Result<Self> {
1161        let state = RunState::load(id)?;
1162        crate::pr_head::ensure_owner(&state)?;
1163        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1164            bail!(
1165                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1166            );
1167        }
1168        if let Some(to) = &state.released_to {
1169            bail!(
1170                "run {} cannot be resumed: its worktree was released to run {}",
1171                state.short(),
1172                crate::run::short_of(to)
1173            );
1174        }
1175        let roles = state.config.resolve_roles()?;
1176        if let Some(id) = &state.config.graph.selected_implementer
1177            && state.candidates.iter().any(|c| c.agent != *id)
1178        {
1179            bail!(
1180                "replacement implementer pin does not match the saved candidates; task must stay held"
1181            );
1182        }
1183        let max_parallel = state.config.graph.max_parallel.max(1);
1184        Ok(Self {
1185            state,
1186            roles,
1187            sem: Arc::new(Semaphore::new(max_parallel)),
1188            pause: Pause::new(),
1189            interrupt: Pause::new(),
1190        })
1191    }
1192
1193    /// Walk the graph to a terminal state, skipping nodes already recorded.
1194    ///
1195    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1196    /// the phone - ends here, so this is the one place a run that ended
1197    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1198    /// notification centre. Best-effort: see [`crate::notices::raise`].
1199    pub async fn execute(&mut self) -> Result<()> {
1200        if let Some(selection) = &self.state.pr_recovery
1201            && (selection.source_proof.is_some() || selection.source_run.is_some())
1202        {
1203            Box::pin(selection.verify_source(&self.state.repo)).await?;
1204        }
1205        // Queue claims cover daemon work; this also serializes manual drivers
1206        // consuming a repair answer, including the new review and decision.
1207        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1208            || self.state.land_approval.is_some()
1209            || self.state.merge_decision.is_some()
1210            || !self.state.merge_repairs.is_empty()
1211            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1212        {
1213            let claim = FixClaim::acquire(&self.state.dir())?;
1214            let disk = RunState::load(&self.state.id)?;
1215            let checkpoint = |s: &RunState| {
1216                (
1217                    s.merge_repairs.len(),
1218                    s.merge_repairs
1219                        .last()
1220                        .map(|r| (r.question.clone(), r.phase)),
1221                    s.land_approval
1222                        .as_ref()
1223                        .map(|a| (a.question.clone(), a.head.clone())),
1224                )
1225            };
1226            if checkpoint(&disk) != checkpoint(&self.state) {
1227                bail!(
1228                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1229                );
1230            }
1231            Some(claim)
1232        } else {
1233            None
1234        };
1235        let mut execution = self.claim_driver()?;
1236        let runner = &mut *execution.runner;
1237        let wait = runner.repository_wait();
1238        // Keep the graph's large future off the callers' stack, including
1239        // standalone task attempts running on a default-sized test thread.
1240        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1241        let result = match result {
1242            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1243                record_repository_block(&mut runner.state, &e);
1244                runner.park_here()?;
1245                Ok(())
1246            }
1247            other => other,
1248        };
1249        execution.driver.finish(&mut runner.state)?;
1250        let ended = if result.is_err() {
1251            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1252        } else {
1253            crate::notices::run_ended(&runner.state)
1254        };
1255        if let Some(notice) = ended {
1256            crate::notices::raise(notice);
1257        }
1258        result
1259    }
1260
1261    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1262        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1263    }
1264
1265    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1266        let wait = self.repository_wait();
1267        coordinate_operation(&mut self.state, cwd, &wait).await
1268    }
1269
1270    /// Validate ownership and persist the driver claim in one critical section.
1271    fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1272        let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1273            Ok(owner) => owner,
1274            Err(error) => {
1275                let detail = format!("worktree maintenance or ownership update: {error:#}");
1276                // Admission did not own run.json: retain this resource outcome
1277                // in memory for queue settlement without overwriting a concurrent
1278                // retirement or publication transaction.
1279                self.state.resource_block = Some(crate::run::ResourceBlock {
1280                    node: "start".into(),
1281                    kind: "worktree maintenance".into(),
1282                    detail: detail.clone(),
1283                    at: Timestamp::now(),
1284                });
1285                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1286            }
1287        };
1288        let disk = RunState::load(&self.state.id)?;
1289        crate::pr_head::ensure_owner(&disk)?;
1290        if disk.liveness(false) == crate::run::Liveness::Live {
1291            bail!("run already has a live driver; wait for its execution to end");
1292        }
1293        if let Some(to) = &disk.released_to {
1294            bail!(
1295                "run {} cannot continue: its worktree was released to run {}",
1296                self.state.short(),
1297                crate::run::short_of(to)
1298            );
1299        }
1300        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1301        let mut execution = DriverExecution {
1302            runner: self,
1303            driver,
1304        };
1305        // Declare after the execution guard: on any claim error this short
1306        // lock must drop before that guard reloads and releases ownership.
1307        let _owner = owner;
1308        let state = &mut execution.runner.state;
1309        // Moving again, so it is no longer parked. Set before the walk rather
1310        // than in `resume`, so every way of re-entering the graph clears it
1311        // and a card cannot claim a run is waiting to be resumed while the
1312        // agents are already working.
1313        state.parked = false;
1314        state.resource_wait = None;
1315        state.resource_block = None;
1316        // Any seat this state still lists as answering belongs to whatever
1317        // process last drove this run — this one included, if it crashed
1318        // mid-wave. Cleared and flushed immediately, before anything else
1319        // runs, so a resume can never show a seat as live when nothing is
1320        // asking it anything yet; the node that actually dispatches the next
1321        // wave repopulates it.
1322        state.clear_active();
1323        // Recorded in the same spot, and flushed together with the clear
1324        // above: this is the pid a reader checks (`RunState::liveness`) when
1325        // no daemon claim exists to answer "is a process still driving this
1326        // run" — a plain `magi run` / `magi review` typed into a terminal
1327        // claims nothing there. Always overwritten, never only-if-absent, so
1328        // a resumed run's stale pid from a previous, possibly-dead process
1329        // can never survive into this one's own report. Unlike
1330        // `clear_active`, this changes on every single `execute()` call, so
1331        // the save below is now unconditional rather than only-if-cleared.
1332        //
1333        // `driver_started_at` is recorded in the same breath, from this same
1334        // pid, so `liveness` can tell a live pid that is genuinely still us
1335        // apart from one the OS has since handed to an unrelated process —
1336        // see that field's own doc for why the pid alone is not enough.
1337        let pid = std::process::id();
1338        state.driver_pid = Some(pid);
1339        state.driver_started_at = crate::proc::process_started_at(pid);
1340        state.driver_exited = false;
1341        state.begin_pass();
1342        execution.driver.bind(state);
1343        state.save()?;
1344        execution.driver.record(state)?;
1345        Ok(execution)
1346    }
1347
1348    async fn execute_graph(&mut self) -> Result<()> {
1349        self.state.configure_recovery_resume()?;
1350        // Preservation may invalidate an old empty classification. Resolve it
1351        // before any resume shortcut can reuse a verdict or publish a branch.
1352        self.persist_candidates().await?;
1353        if self
1354            .state
1355            .merge_decision
1356            .as_ref()
1357            .is_some_and(|decision| !decision.closed)
1358            && self.state.status != RunStatus::Landing
1359        {
1360            if crate::merge_decision::resumable_hold(&self.state) {
1361                self.state.merge = Some(MergeOutcome {
1362                    mode: MergeMode::Pr,
1363                    ok: false,
1364                    detail: self
1365                        .state
1366                        .merge_decision
1367                        .as_ref()
1368                        .unwrap()
1369                        .target
1370                        .pr
1371                        .clone(),
1372                    empty: false,
1373                });
1374                self.run_land().await?;
1375                self.settle_questions();
1376                return Ok(());
1377            }
1378            bail!(
1379                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1380            );
1381        }
1382        if self.state.config.graph.selected_implementer.is_some()
1383            && self.state.status == RunStatus::Blocked
1384            && self.state.tally.is_none()
1385            && self.state.candidate_preservation.is_empty()
1386            && self.state.interrupted_implementations.is_empty()
1387        {
1388            bail!(
1389                "selected implementation is held; explicitly release the owning task to retry it"
1390            );
1391        }
1392
1393        if self.state.merge_repairs.last().is_some_and(|r| {
1394            matches!(
1395                r.phase,
1396                crate::run::MergeRepairPhase::Requested
1397                    | crate::run::MergeRepairPhase::Fixing
1398                    | crate::run::MergeRepairPhase::Reviewing
1399            )
1400        }) {
1401            if self.repair_merge_review().await? {
1402                self.run_land().await?;
1403            }
1404            self.settle_questions();
1405            return Ok(());
1406        }
1407        // Publication already passed review and gate. Resume only that node:
1408        // syncing to a newer base here could change the verified commit.
1409        if self.state.pr_publication_pending {
1410            self.merge().await?;
1411            if self.resource_halt()? {
1412                return Ok(());
1413            }
1414            self.state.save()?;
1415            self.settle_questions();
1416            return Ok(());
1417        }
1418        // A run that already lost its quorum never resumes into the verdict
1419        // machinery: `deliberate` and `vote` would otherwise clobber the
1420        // stalled marker back to Voting and the run would keep going past a
1421        // verdict that is no longer trustworthy. Everything already recorded is
1422        // kept, so the run stays resumable (or foldable) for a human to pick up.
1423        //
1424        // On --resume the run gets one chance to repair itself: the seats a
1425        // rate limit took out are re-asked. If their quota has since reset and
1426        // the quorum is restored, the run picks up and finishes; otherwise it
1427        // stays stale and still-resumable for a later retry. If it does not
1428        // recover, the returned status stays `Stalled` and nothing was
1429        // clobbered (the recovery only mutates entries for the lost seats).
1430        if self.state.status == RunStatus::Stalled {
1431            if self.recover_stall().await? {
1432                self.finish_after_tally().await?;
1433            } else {
1434                // Still below quorum: persist the marker and stay resumable.
1435                self.state.save()?;
1436            }
1437            return Ok(());
1438        }
1439        // A run parked inside `land` - watching CI, mid fix-round, or
1440        // waiting on the owner's merge approval - resumes directly into it,
1441        // never back through `prep`. Everything before `merge` already
1442        // concluded; that is the only way `status` reaches `Landing` in the
1443        // first place. Re-walking `review_loop` first would also be actively
1444        // wrong: its own status recomputation (see its doc) treats any
1445        // clean round as reason to set `status` to `Gating`, which would
1446        // clobber this marker before `merge` ever ran, and this run would
1447        // never find its way back into `land` at all.
1448        if self.state.status == RunStatus::Landing {
1449            self.run_land().await?;
1450            // `run_land` may have settled the run right here - CI came back
1451            // green and the PR merged, say - without ever passing back
1452            // through `merge`'s own trailing call. Whatever it left `status`
1453            // as is what this has to read.
1454            self.settle_questions();
1455            return Ok(());
1456        }
1457        // A run parked at `merge` on the owner's word about a withheld PR text
1458        // resumes straight into `merge`. Re-walking the tail would sync to the
1459        // base again (possibly rebasing, possibly spending the fixer) after the
1460        // gate had passed, and then push a tree nobody reviewed or gated.
1461        if self.state.github_text.is_some()
1462            && self.state.merge.is_none()
1463            && self.state.status == RunStatus::Gating
1464        {
1465            self.merge().await?;
1466            if self.resource_halt()? {
1467                return Ok(());
1468            }
1469            self.state.save()?;
1470            return Ok(());
1471        }
1472        if let Err(e) = self.prep().await {
1473            if e.is::<crate::repo_lock::ResourceBlocked>() {
1474                return Err(e);
1475            }
1476            self.state.status = RunStatus::Blocked;
1477            self.state
1478                .prep_failure
1479                .get_or_insert_with(|| format!("{e:#}"));
1480            self.state
1481                .event("prep", format!("preparation stopped: {e:#}"));
1482            self.state.save()?;
1483            return Err(e);
1484        }
1485        if self.park_here()? {
1486            return Ok(());
1487        }
1488        self.advise().await?;
1489        if self.park_here()? {
1490            return Ok(());
1491        }
1492        self.implement().await?;
1493        if self.park_here()? {
1494            return Ok(());
1495        }
1496        // `after_implement` already saved the state and settled any open
1497        // questions when it set this; nothing later in the graph has
1498        // anything to judge.
1499        if self.state.status == RunStatus::VerifiedNoop {
1500            return Ok(());
1501        }
1502        self.judge().await?;
1503        if self.park_here()? {
1504            return Ok(());
1505        }
1506        self.deliberate().await?;
1507        if self.park_here()? {
1508            return Ok(());
1509        }
1510        self.vote().await?;
1511        if self.park_here()? {
1512            return Ok(());
1513        }
1514        self.tally()?;
1515        // A verdict that lost its quorum is not trustworthy: do not review,
1516        // gate, or merge on it. Everything already done is kept, so the run
1517        // stays resumable (or foldable); the human can replace the agent that
1518        // ran out of quota and pick it up.
1519        if self.state.status == RunStatus::Stalled {
1520            // Persist the stalled marker now — the normal end-of-execute save
1521            // below is below this early return, and without it a resumed run
1522            // would reload a pre-tally status and keep going.
1523            self.state.save()?;
1524            return Ok(());
1525        }
1526        self.finish_after_tally().await?;
1527        Ok(())
1528    }
1529
1530    /// A wave could not start its agents because a shared resource stayed busy
1531    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1532    /// resumable. Unstarted seats have no result; completed peers retain
1533    /// their saved candidate state.
1534    fn stop_for_resource(&mut self) -> Result<()> {
1535        // A wait abandoned because the run is parking stays a park.
1536        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1537        {
1538            self.state.status = RunStatus::Blocked;
1539        }
1540        self.state.save()
1541    }
1542
1543    /// Stop walking the graph if a node ended on a resource block.
1544    fn resource_halt(&mut self) -> Result<bool> {
1545        if self.state.resource_block.is_none() {
1546            return Ok(false);
1547        }
1548        self.stop_for_resource()?;
1549        Ok(true)
1550    }
1551
1552    /// Park here if asked to, recording it in the run's own timeline.
1553    ///
1554    /// Returns whether the caller should stop walking the graph. The state is
1555    /// saved either way by the node that just finished; this adds the event so
1556    /// the operator's card says why a run that is neither finished nor moving
1557    /// is sitting where it is.
1558    fn park_here(&mut self) -> Result<bool> {
1559        let parking = self.pause.parked() || self.interrupt.parked();
1560        if !parking && self.state.resource_block.is_some() {
1561            self.stop_for_resource()?;
1562            return Ok(true);
1563        }
1564        // Either handle asking is enough - see `Pause`'s own doc for why
1565        // they are never the same one. `interrupt` is checked second so a
1566        // reason it carries is preferred in the message below over a plain
1567        // shutdown park racing it at the same boundary.
1568        if !self.pause.parked() && !self.interrupt.parked() {
1569            return Ok(false);
1570        }
1571        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1572        let why = match cause.as_ref().map(ParkCause::label) {
1573            Some(reason) => format!(
1574                "parked after `{}` ({reason}) — resume to carry on from here",
1575                self.state.status.as_str()
1576            ),
1577            None => format!(
1578                "parked after `{}` — resume to carry on from here",
1579                self.state.status.as_str()
1580            ),
1581        };
1582        self.state.event("park", why);
1583        self.state
1584            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1585        self.state.parked = true;
1586        self.state.save()?;
1587        Ok(true)
1588    }
1589
1590    /// Only a new explicit operator approval may retry failed preparation.
1591    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1592        if self.state.prep_failure.is_none() {
1593            return Ok(());
1594        }
1595        let failed_at = self
1596            .state
1597            .events
1598            .iter()
1599            .rev()
1600            .find(|event| event.node == "prep")
1601            .map_or(self.state.updated_at, |event| event.at);
1602        anyhow::ensure!(
1603            approval_at > failed_at,
1604            "preparation retry requires a new explicit release after the failure"
1605        );
1606        self.state.prep_failure = None;
1607        self.state.prep_complete = Some(false);
1608        self.state
1609            .event("prep-retry", "operator approved retry of preparation");
1610        Ok(())
1611    }
1612
1613    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1614    pub fn on_pause(&mut self, pause: Pause) {
1615        self.pause = pause;
1616    }
1617
1618    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1619    /// scheduler asking this one run - and no other - to park so a task
1620    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1621    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1622    /// handle.
1623    pub fn watch_interrupt(&mut self, pause: Pause) {
1624        self.interrupt = pause;
1625    }
1626
1627    /// Abandon this run's own open questions, once `status` has actually
1628    /// settled rather than merely paused.
1629    ///
1630    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1631    /// either back up with the candidates, the review round and the seat
1632    /// sessions already on disk, so a question an implementer asked mid-round
1633    /// may still get a real answer read by a real resume. Only the statuses
1634    /// `resumable` excludes are actually final: the run merged, it reached
1635    /// `Ready` with nothing left to do, it failed outright with no
1636    /// established point to continue from, or every candidate agreed, with
1637    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1638    /// every one of those the seat that asked is gone for good, exactly like
1639    /// the run being deleted under `magi run rm` - so the same cleanup
1640    /// applies, worded for what actually happened instead of "the run was
1641    /// deleted".
1642    ///
1643    /// Best-effort and silent on success: called from every place `status`
1644    /// can land on one of those three, including ones a resumed run revisits,
1645    /// so it must cost nothing when there was nothing open to begin with.
1646    fn settle_questions(&mut self) {
1647        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1648            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1649        }
1650    }
1651
1652    /// The tail of the graph after a trustworthy tally: fold losers, review,
1653    /// gate, merge, and persist.
1654    async fn finish_after_tally(&mut self) -> Result<()> {
1655        // Before the losers are folded: their branches are what the graft
1656        // reads from.
1657        self.graft().await?;
1658        if self.resource_halt()? {
1659            return Ok(());
1660        }
1661        self.fold_losers().await?;
1662        // Before review starts, and again right before the gate: a run's
1663        // review rounds can themselves take long enough for the base to move
1664        // a second time, and the gate is the one node whose "green" gets
1665        // acted on.
1666        self.sync_to_base().await?;
1667        if self.resource_halt()? {
1668            return Ok(());
1669        }
1670        if self.state.status == RunStatus::AlreadyInBase {
1671            return Ok(());
1672        }
1673        self.review_loop().await?;
1674        if self.resource_halt()? {
1675            return Ok(());
1676        }
1677        self.sync_to_base().await?;
1678        if self.resource_halt()? {
1679            return Ok(());
1680        }
1681        if self.state.status == RunStatus::AlreadyInBase {
1682            return Ok(());
1683        }
1684        self.gate().await?;
1685        self.merge().await?;
1686        if self.resource_halt()? {
1687            return Ok(());
1688        }
1689        self.state.save()?;
1690        Ok(())
1691    }
1692
1693    // ---------------------------------------------------------------- prep
1694
1695    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1696    /// run with the step and its output in the event; it is never swallowed.
1697    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1698    /// resume starts over instead of finding candidates and missing seats.
1699    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1700        let result = match ensure_setup_config(&mut self.state, repo).await {
1701            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1702            Err(e) => Err(e),
1703        };
1704        if let Err(e) = result {
1705            git::worktree_remove_clean(repo, wt).await.ok();
1706            if in_prep {
1707                self.rollback_prep(repo).await;
1708            }
1709            self.state.status = RunStatus::Blocked;
1710            self.state.event(
1711                "setup",
1712                format!("worktree setup failed in {}: {e:#}", wt.display()),
1713            );
1714            self.state.save()?;
1715            return Err(e);
1716        }
1717        Ok(())
1718    }
1719
1720    /// Undo everything `prep` created, so a resume re-runs it whole.
1721    async fn rollback_prep(&mut self, repo: &Path) {
1722        let root = self.state.worktree_root();
1723        for c in std::mem::take(&mut self.state.candidates) {
1724            if matches!(
1725                git::worktree_remove_clean(repo, &c.worktree).await,
1726                Ok(true)
1727            ) {
1728                git::branch_delete(repo, &c.branch).await.ok();
1729            } else {
1730                self.state.candidates.push(c);
1731            }
1732        }
1733        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1734        for j in 1..=seats {
1735            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1736                .await
1737                .ok();
1738        }
1739        for k in 1..=self.state.config.graph.advisors {
1740            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1741                .await
1742                .ok();
1743        }
1744    }
1745
1746    async fn prep(&mut self) -> Result<()> {
1747        if let Some(reason) = &self.state.prep_failure {
1748            bail!(
1749                "candidate preparation previously failed: {reason}; correct the source before retrying"
1750            );
1751        }
1752        if self
1753            .state
1754            .prep_complete
1755            .unwrap_or(!self.state.candidates.is_empty())
1756        {
1757            return Ok(());
1758        }
1759        let repo = self.state.repo.clone();
1760        // Includes rollback, config, refs, checkout creation and setup. No
1761        // repository mutation may precede the real shared lock acquisition.
1762        let _coordination = self.coordinate(&repo).await?;
1763        self.rollback_prep(&repo).await;
1764        if !self.state.candidates.is_empty() {
1765            bail!(
1766                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1767            );
1768        }
1769        self.state.prep_complete = Some(false);
1770        self.state.status = RunStatus::Prep;
1771        self.state.save()?;
1772        let repo = self.state.repo.clone();
1773        let base = self.state.base_commit.clone();
1774        let plan = refs::plan(&repo, &self.state.seeds).await?;
1775        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1776        let root = self.state.worktree_root();
1777        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1778
1779        // The hook is the write-time half of the blindness contract; the
1780        // presentation filter in `blind` is the half that cannot be bypassed.
1781        let hooks_dir = self.state.dir().join("hooks");
1782        if self.state.config.blind.commit_msg_hook {
1783            std::fs::create_dir_all(&hooks_dir)
1784                .with_context(|| format!("create {}", hooks_dir.display()))?;
1785            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1786            let path = hooks_dir.join("commit-msg");
1787            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1788            make_executable(&path)?;
1789            // Ref-counted rather than a plain idempotent set: with more than
1790            // one run able to be in flight in the same repository at once
1791            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1792            // true?" check cannot tell "another run of mine still needs
1793            // this" from "nobody does", and the run that happens to finish
1794            // first would disable the hook out from under a sibling still
1795            // relying on it.
1796            git::acquire_worktree_config(&repo).await?;
1797            self.state.enabled_worktree_config = true;
1798        }
1799
1800        // `[worktree] setup` hides its products per worktree, which needs the
1801        // same per-worktree config the hook does. Held until fold, like it.
1802        ensure_setup_config(&mut self.state, &repo).await?;
1803
1804        for (index, (spec, label)) in self
1805            .roles
1806            .implementers
1807            .clone()
1808            .into_iter()
1809            .zip(labels)
1810            .enumerate()
1811        {
1812            let branch = self.state.branch_for(label);
1813            let worktree = root.join(format!("cand-{label}"));
1814            // Persist intent before Git can leave a partially created tree.
1815            // Candidate registration follows setup, so it cannot inventory
1816            // crashes or refusals during creation itself.
1817            self.state.event(
1818                "prep-intent",
1819                serde_json::json!({
1820                    "branch": branch,
1821                    "worktree": worktree,
1822                    "start": start,
1823                    "picks": plan.picks,
1824                })
1825                .to_string(),
1826            );
1827            self.state.save()?;
1828            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1829            if self.state.config.blind.commit_msg_hook {
1830                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1831            }
1832            git::local_exclude(&worktree, "/.magi/").await?;
1833            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1834                // Nothing of this prep may survive: a resume must start over,
1835                // not find half the candidates and skip the rest.
1836                git::worktree_remove_clean(&repo, &worktree).await.ok();
1837                git::branch_delete(&repo, &branch).await.ok();
1838                self.rollback_prep(&repo).await;
1839                self.state.status = RunStatus::Blocked;
1840                self.state
1841                    .event("prep", format!("worktree setup failed: {e:#}"));
1842                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1843                self.state.save()?;
1844                return Err(e);
1845            }
1846            self.state.candidates.push(Candidate {
1847                index,
1848                label,
1849                agent: spec.id.clone(),
1850                branch,
1851                worktree,
1852                summary: String::new(),
1853                stat: String::new(),
1854                files: 0,
1855                commits: 0,
1856                empty: false,
1857                failed: None,
1858                verified_noop: None,
1859                duration_ms: 0,
1860                folded: false,
1861            });
1862            self.state.save()?;
1863            for pick in &plan.picks {
1864                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1865                {
1866                    self.state.status = RunStatus::Blocked;
1867                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1868                    self.state
1869                        .event("prep", format!("cannot apply referenced commit: {e}"));
1870                    self.state.save()?;
1871                    return Err(e);
1872                }
1873            }
1874        }
1875
1876        for j in 1..=self.roles.judges.len() {
1877            let wt = root.join(format!("judge-{j}"));
1878            if !wt.exists() {
1879                git::worktree_add_detached(&repo, &wt, &base).await?;
1880                self.setup_seat_worktree(&repo, &wt, true).await?;
1881            }
1882        }
1883
1884        // Disposable, detached checkouts for the design-deliberation stage's
1885        // advisor seats — the same shape as the judges' above, at the same
1886        // base commit, since advisors also only ever read. Sized off the
1887        // configured count directly rather than a resolved roster: unlike
1888        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1889        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1890        // `prep` has no `ResolvedRoles` field to read a count from here.
1891        if self.state.config.graph.advise {
1892            for k in 1..=self.state.config.graph.advisors {
1893                let wt = root.join(format!("advisor-{k}"));
1894                if !wt.exists() {
1895                    git::worktree_add_detached(&repo, &wt, &base).await?;
1896                    self.setup_seat_worktree(&repo, &wt, true).await?;
1897                }
1898            }
1899        }
1900
1901        // A judge cannot tell it is looking at its own patch — the seats keep
1902        // separate conversations — but a panel that shares agents with the
1903        // field is less independent than it looks, and that is worth saying out
1904        // loud once per run rather than leaving it in the config.
1905        let authors: Vec<&str> = self
1906            .roles
1907            .implementers
1908            .iter()
1909            .map(|a| a.id.as_str())
1910            .collect();
1911        let overlap: Vec<String> = self
1912            .roles
1913            .judges
1914            .iter()
1915            .enumerate()
1916            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1917            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1918            .collect();
1919        if !overlap.is_empty() {
1920            let note = format!(
1921                "{} also authored a candidate; blind, but the panel is less \
1922                 independent than {} distinct agents would be",
1923                overlap.join(", "),
1924                self.roles.judges.len()
1925            );
1926            self.state.event("prep", note);
1927        }
1928
1929        self.state.event(
1930            "prep",
1931            format!(
1932                "{} candidates, {} judges, base {} ({})",
1933                self.state.candidates.len(),
1934                self.roles.judges.len(),
1935                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1936                self.state.base_branch
1937            ),
1938        );
1939        self.state.prep_complete = Some(true);
1940        self.state.status = RunStatus::Implementing;
1941        self.state.save()?;
1942        Ok(())
1943    }
1944
1945    // -------------------------------------------------------------- advise
1946
1947    /// The design-deliberation stage: independent, read-only advisor seats
1948    /// each sketch a design before any implementer touches the repository,
1949    /// and (when at least one produced a usable proposal) a synthesis seat
1950    /// blends them into a brief `implement` carries in every candidate's
1951    /// prompt.
1952    ///
1953    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1954    /// advisors` is the proposal count. Everything here is best-effort and
1955    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1956    /// that cannot reach quota, or a synthesis seat that produced nothing
1957    /// usable all leave `implement` exactly as it was before this stage
1958    /// existed — the task instruction alone — rather than failing the whole
1959    /// competition over an enrichment stage. Every outcome is still recorded
1960    /// as an event, so a run that got nothing from this stage says why.
1961    ///
1962    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1963    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1964    /// resumed run whose stage failed would re-run it, and re-spend the
1965    /// agent calls, on every reentry before `implement`.
1966    ///
1967    /// Also skipped once any candidate shows implementation progress — the
1968    /// exact predicate `implement` itself uses to decide a candidate is no
1969    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1970    /// is not enough: a run created by an older binary that predates this
1971    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1972    /// an already-`Implementing`-or-later run under this build would
1973    /// otherwise walk straight back through `prep` (a no-op once candidates
1974    /// exist) into this node and spawn every advisor seat against worktrees
1975    /// `prep` never recreated — after implementation has already started,
1976    /// which is exactly the invariant this stage exists to guarantee.
1977    async fn advise(&mut self) -> Result<()> {
1978        let implement_untouched = self
1979            .state
1980            .candidates
1981            .iter()
1982            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1983        if !self.state.config.graph.advise || self.state.advise_attempted {
1984            return Ok(());
1985        }
1986        if !implement_untouched {
1987            self.state.event(
1988                "advise",
1989                "skipping the design-deliberation stage: at least one \
1990                 candidate already shows implementation progress, so this \
1991                 run is past the point the stage exists to run before"
1992                    .to_owned(),
1993            );
1994            self.state.advise_attempted = true;
1995            self.state.save()?;
1996            return Ok(());
1997        }
1998        let run_id = self.state.id.clone();
1999        let prompts = self.state.config.prompts.clone();
2000        let instruction = self.state.instruction.clone();
2001        let language = self.state.config.graph.language.clone();
2002        let root = self.state.worktree_root();
2003        let n = self.state.config.graph.advisors;
2004        let where_recorded = self.state.dir().join("run.json");
2005
2006        let seats = match self.state.config.advisors() {
2007            Ok(seats) if !seats.is_empty() => seats,
2008            Ok(_) => {
2009                self.state.event(
2010                    "advise",
2011                    format!(
2012                        "[graph] advisors is 0; skipping the design-deliberation \
2013                         stage and continuing without a synthesis brief (see {})",
2014                        where_recorded.display()
2015                    ),
2016                );
2017                self.state.advise_attempted = true;
2018                self.state.save()?;
2019                return Ok(());
2020            }
2021            Err(e) => {
2022                self.state.event(
2023                    "advise",
2024                    format!(
2025                        "could not resolve advisor seats ({e:#}); continuing \
2026                         without a design-deliberation brief (see {})",
2027                        where_recorded.display()
2028                    ),
2029                );
2030                self.state.advise_attempted = true;
2031                self.state.save()?;
2032                return Ok(());
2033            }
2034        };
2035
2036        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2037        let artifacts = agent::artifacts_dir(&self.state.dir());
2038        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2039
2040        let mut jobs = Vec::new();
2041        for (i, spec) in seats.iter().cloned().enumerate() {
2042            let seat_key = format!("advisor-{}", i + 1);
2043            let seat = self.seat(&seat_key, &spec.id);
2044            jobs.push(SeatJob {
2045                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2046                spec,
2047                seat,
2048                cwd: worktrees[i % worktrees.len()].clone(),
2049                timeout,
2050                allow_write: false,
2051                sessions: false,
2052                artifacts: artifacts.clone(),
2053                stem: seat_key,
2054                handover: None,
2055            });
2056        }
2057
2058        self.state.event(
2059            "advise",
2060            format!(
2061                "{} advisor seat(s) sketching a design in parallel",
2062                jobs.len()
2063            ),
2064        );
2065        let mut quota_losses = Vec::new();
2066        let cache = self.state.config.cache_dir();
2067        let ctx = WaveCtx {
2068            carry_seats: false,
2069            pauses: Some((&self.pause, &self.interrupt)),
2070            run: &run_id,
2071            node: "advise",
2072            prompts: &prompts,
2073            cache: cache.as_deref(),
2074            round: None,
2075        };
2076        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2077        let results = ask_json_wave::<Proposal>(
2078            jobs,
2079            Arc::clone(&self.sem),
2080            self.state.config.graph.retries,
2081            &advisor_roster,
2082            &ctx,
2083            &mut quota_losses,
2084            &mut self.state,
2085            &|p: &Proposal| p.validate(),
2086        )
2087        .await;
2088        if self.state.resource_block.is_some() {
2089            return self.stop_for_resource();
2090        }
2091        self.state.quota.extend(quota_losses);
2092
2093        let mut records = Vec::with_capacity(results.len());
2094        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2095            let agent_id = seat.agent.clone();
2096            self.state.seats.insert(seat.key.clone(), seat);
2097            match res {
2098                Ok((proposal, out)) => {
2099                    self.state
2100                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2101                    records.push(advise::AdvisorRecord::proposed(
2102                        i + 1,
2103                        agent_id,
2104                        proposal,
2105                        out.duration_ms,
2106                    ));
2107                }
2108                Err(e) => {
2109                    self.state.event(
2110                        "advise",
2111                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2112                    );
2113                    records.push(advise::AdvisorRecord::failed(
2114                        i + 1,
2115                        agent_id,
2116                        e.to_string(),
2117                    ));
2118                }
2119            }
2120        }
2121
2122        let mut advice = advise::Advice {
2123            records,
2124            synthesis: None,
2125        };
2126        if advice.proposals().is_empty() {
2127            self.state.event(
2128                "advise",
2129                "no advisor produced a usable proposal; continuing without a \
2130                 synthesis brief"
2131                    .to_owned(),
2132            );
2133        } else {
2134            match self
2135                .synthesize_brief(
2136                    &advice,
2137                    &instruction,
2138                    &language,
2139                    &worktrees[0],
2140                    &artifacts,
2141                    &run_id,
2142                    &prompts,
2143                    cache.as_deref(),
2144                )
2145                .await
2146            {
2147                Ok(Some(text)) => {
2148                    self.state.event(
2149                        "advise",
2150                        "synthesized a design brief for the implementer".to_owned(),
2151                    );
2152                    advice.synthesis = Some(text);
2153                }
2154                Ok(None) => {
2155                    self.state.event(
2156                        "advise",
2157                        "the synthesis seat produced nothing usable; continuing \
2158                         without a design brief"
2159                            .to_owned(),
2160                    );
2161                }
2162                Err(e) => {
2163                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2164                        return Err(e);
2165                    }
2166                    self.state.event(
2167                        "advise",
2168                        format!("could not synthesize a design brief: {e:#}"),
2169                    );
2170                }
2171            }
2172        }
2173        advise::apply_reflection(&mut advice);
2174
2175        self.state.advice = Some(advice);
2176        self.state.advise_attempted = true;
2177        self.state.save()?;
2178        Ok(())
2179    }
2180
2181    /// The synthesis seat: reads every advisor's proposal and blends them
2182    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2183    /// out of [`Runner::advise`] only for readability — it is not called
2184    /// anywhere else.
2185    ///
2186    /// Picked the same way [`crate::talk`]'s standing conversation and
2187    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2188    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2189    /// order (a claude seat, else the first runnable agent in roster order)
2190    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2191    /// in [`crate::config`] for why a dedicated field exists here at all.
2192    #[allow(clippy::too_many_arguments)]
2193    async fn synthesize_brief(
2194        &mut self,
2195        advice: &advise::Advice,
2196        instruction: &str,
2197        language: &str,
2198        cwd: &Path,
2199        artifacts: &Path,
2200        run_id: &str,
2201        prompts: &Prompts,
2202        cache: Option<&Path>,
2203    ) -> Result<Option<String>> {
2204        let chain = agent::pick_chain(
2205            &self.state.config.agents,
2206            self.state.config.roles.synthesizer.as_ref(),
2207            &agent::installed,
2208            "synthesizer",
2209        )?;
2210        let proposals = advice.proposals();
2211        let mut prompt = prompt::with_overlay(
2212            prompt::synthesize_brief(instruction, &proposals, language),
2213            prompts.overlay("advise"),
2214        );
2215        if cache.is_some() {
2216            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2217            // below — see `prompt::build_cache_note`'s doc for why telling a
2218            // read-only seat to build through the shared cache is exactly how
2219            // a sandbox's write refusal gets misread as a defect.
2220            prompt.push('\n');
2221            prompt.push_str(&prompt::build_cache_note("advise", false));
2222        }
2223        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2224        // Each id is tried once, in order; a quota hit, error or unusable
2225        // answer moves to the next. The seat is single-turn (`sessions:
2226        // false`) and the prompt is the whole context, so a fallback agent
2227        // needs nothing carried over.
2228        let mut last = None;
2229        for (n, spec) in chain.iter().enumerate() {
2230            if n > 0 {
2231                self.state
2232                    .event("advise", format!("synthesis falling back to {}", spec.id));
2233            }
2234            let mut seat = self.seat("advise-synthesis", &spec.id);
2235            let wait = self.repository_wait();
2236            let outcome = wait
2237                .scope(agent::invoke(
2238                    spec,
2239                    &mut seat,
2240                    &Invocation {
2241                        scope: crate::agent::InvocationScope::Repository,
2242                        coordination: self.state.config.coordination.as_ref(),
2243                        fetched_base: None,
2244                        cwd,
2245                        prompt: &prompt,
2246                        timeout,
2247                        allow_write: false,
2248                        unsandboxed: false,
2249                        sessions: false,
2250                        artifacts,
2251                        stem: &if n == 0 {
2252                            "advise-synthesis".to_owned()
2253                        } else {
2254                            format!("advise-synthesis-{}", spec.id)
2255                        },
2256                        run: run_id,
2257                        node: "advise",
2258                        cache_dir: None,
2259                        attachments: &[],
2260                        writable: &[],
2261                    },
2262                ))
2263                .await;
2264            if outcome.is_ok() {
2265                self.state.seats.insert(seat.key.clone(), seat);
2266            }
2267            let advance = agent::chain_advances(&outcome);
2268            last = Some(outcome);
2269            if !advance {
2270                break;
2271            }
2272        }
2273        // Exhausted: the last attempt's result is what a single failed seat
2274        // would have produced.
2275        let out = last.expect("a chain holds at least one agent")?;
2276        if !out.usable() {
2277            return Ok(None);
2278        }
2279        let text =
2280            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2281        Ok((!text.trim().is_empty()).then_some(text))
2282    }
2283
2284    // ----------------------------------------------------------- implement
2285
2286    async fn implement(&mut self) -> Result<()> {
2287        self.persist_candidates().await?;
2288        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2289        // agent files with `magi task add` name the run that paid for it. The
2290        // prompt overlay is cloned alongside it because the waves borrow it
2291        // while `self` is mutably borrowed by the node's own bookkeeping.
2292        let run_id = self.state.id.clone();
2293        let prompts = self.state.config.prompts.clone();
2294        let todo: Vec<usize> = self
2295            .state
2296            .candidates
2297            .iter()
2298            .enumerate()
2299            .filter(|(_, c)| {
2300                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2301                    && c.failed.is_none()
2302                    && !c.empty
2303            })
2304            .map(|(i, _)| i)
2305            .collect();
2306        if todo.is_empty() {
2307            return self.after_implement();
2308        }
2309        self.state.status = RunStatus::Implementing;
2310
2311        let language = self.state.config.graph.language.clone();
2312        let github_language = self.state.config.graph.github_language.clone();
2313        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2314        let sessions = self.state.config.graph.sessions;
2315        let artifacts = agent::artifacts_dir(&self.state.dir());
2316        // The design-deliberation stage's blended brief, when `advise` found
2317        // one — carried into every implementer's prompt the same way
2318        // regardless of which candidate it is.
2319        let brief = self
2320            .state
2321            .advice
2322            .as_ref()
2323            .and_then(|a| a.synthesis.as_deref())
2324            .map(str::to_owned);
2325        let attachments = self.state.attachments.clone();
2326
2327        let mut jobs = Vec::new();
2328        for &i in &todo {
2329            let (index, label, worktree) = {
2330                let c = &self.state.candidates[i];
2331                (c.index, c.label, c.worktree.clone())
2332            };
2333            let spec = self.roles.implementers[index].clone();
2334            let seat_key = format!("impl-{label}");
2335            let seat = self.seat(&seat_key, &spec.id);
2336            let instruction = seeded_instruction(&self.state);
2337            jobs.push(SeatJob {
2338                spec,
2339                seat,
2340                prompt: prompt::implement_in(
2341                    &instruction,
2342                    &worktree.to_string_lossy(),
2343                    &language,
2344                    &github_language,
2345                    brief.as_deref(),
2346                    &attachments,
2347                ),
2348                cwd: worktree,
2349                timeout,
2350                allow_write: true,
2351                sessions,
2352                artifacts: artifacts.clone(),
2353                stem: format!("impl-{label}"),
2354                handover: None,
2355            });
2356        }
2357
2358        self.state.event(
2359            "implement",
2360            format!("{} candidates in parallel", jobs.len()),
2361        );
2362        // Kept so a seat whose CLI hung up can be asked again from the same
2363        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2364        // can update a seat's own entry once a fallback agent takes it over —
2365        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2366        // whichever agent actually answered, not the one that quota'd out.
2367        let mut sent = jobs.clone();
2368        let cache = self.state.config.cache_dir();
2369        let ctx = WaveCtx {
2370            carry_seats: false,
2371            pauses: Some((&self.pause, &self.interrupt)),
2372            run: &run_id,
2373            node: "implement",
2374            prompts: &prompts,
2375            cache: cache.as_deref(),
2376            round: None,
2377        };
2378        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2379        if !wave_blocked(&results) {
2380            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2381                .await;
2382            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2383                .await?;
2384            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2385                .await;
2386        }
2387        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2388
2389        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2390            && results
2391                .iter()
2392                .any(|(_, _, out)| FailClass::of(out).is_some());
2393        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2394            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2395                // Executed turns interrupted by disk pressure already carry
2396                // a preservation barrier from wave. Admission-only waits do
2397                // not. Keep the interrupted seat's session for recovery too.
2398                if self
2399                    .state
2400                    .candidate_preservation
2401                    .contains_key(&self.state.candidates[i].label)
2402                {
2403                    self.state.seats.insert(seat.key.clone(), seat);
2404                }
2405                continue;
2406            }
2407            let seat_key = seat.key.clone();
2408            // A quota fallback (`resume_seat_handovers`) may have handed this
2409            // seat to a different agent than the one `prep` recorded on the
2410            // candidate; the stats tables and any later fixer-defaults-to-
2411            // winner's-author lookup must credit whoever actually answered —
2412            // unless every fallback also quota'd out, in which case nobody
2413            // actually answered and crediting the last agent tried would
2414            // erase every earlier agent's own quota loss from the stats
2415            // tables instead of just this one seat's.
2416            let agent = seat.agent.clone();
2417            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2418            self.state.seats.insert(seat.key.clone(), seat);
2419            let label = self.state.candidates[i].label;
2420            self.state.interrupted_implementations.remove(&label);
2421
2422            let (summary, duration, failed, verified_claim) = match out {
2423                AgentOutcome::Ok(o) => {
2424                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2425                    let failed = (!o.usable()).then(|| {
2426                        if o.timed_out {
2427                            "agent timed out".to_owned()
2428                        } else {
2429                            format!(
2430                                "agent exited with {}",
2431                                crate::notices::exit_status(o.exit_code)
2432                            )
2433                        }
2434                    });
2435                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2436                    (text, o.duration_ms, failed, verified_claim)
2437                }
2438                // Left un-resumed by `resume_undelivered` (a dirty tree
2439                // already rescues the work, or there was no session left to
2440                // resume into) — reported like the ordinary failure it is,
2441                // never as if `o.text` (the CLI's raw error JSON) were an
2442                // answer.
2443                AgentOutcome::Dropped(o) => {
2444                    let why = o
2445                        .dropped
2446                        .as_ref()
2447                        .map(|d| d.why.as_str())
2448                        .unwrap_or("the CLI ended the stream without delivering its answer");
2449                    (
2450                        String::new(),
2451                        o.duration_ms,
2452                        Some(format!("the CLI dropped the stream ({why})")),
2453                        None,
2454                    )
2455                }
2456                AgentOutcome::Quota(o) => {
2457                    self.state.quota.push(QuotaLoss {
2458                        seat: seat_key,
2459                        node: "implement".to_owned(),
2460                        at: Timestamp::now(),
2461                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2462                    });
2463                    (
2464                        String::new(),
2465                        o.duration_ms,
2466                        Some("rate limited (quota); produced no change".to_owned()),
2467                        None,
2468                    )
2469                }
2470                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2471                    (String::new(), 0, Some(e), None)
2472                }
2473            };
2474
2475            // Save every completed turn before any fallible Git operation. A failed
2476            // rescue must not erase the reply or prevent collection of other seats.
2477            let c = &mut self.state.candidates[i];
2478            if !exhausted_the_fallback_chain {
2479                c.agent = agent;
2480            }
2481            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2482            c.duration_ms = duration;
2483            c.failed = failed;
2484            c.verified_noop = verified_claim;
2485            self.state
2486                .candidate_preservation
2487                .insert(label, "awaiting persistence".to_owned());
2488        }
2489        self.state.save()?;
2490        self.persist_candidates().await?;
2491
2492        if resource_blocked {
2493            return self.stop_for_resource();
2494        }
2495        if selected_failed {
2496            self.state.status = RunStatus::Blocked;
2497            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2498            self.state.save()?;
2499            bail!(
2500                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2501            );
2502        }
2503        self.after_implement()
2504    }
2505
2506    /// Retry only persistence after a completed implementation turn.
2507    async fn persist_candidates(&mut self) -> Result<()> {
2508        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2509        for label in labels {
2510            let i = self
2511                .state
2512                .candidates
2513                .iter()
2514                .position(|c| c.label == label)
2515                .context("pending candidate is missing")?;
2516            let c = self.state.candidates[i].clone();
2517            let interrupted = self.state.interrupted_implementations.contains(&label);
2518            let result = async {
2519                let wait = crate::repo_lock::WaitPolicy::new(
2520                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2521                    || false,
2522                );
2523                let _coordination =
2524                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2525                // Best effort evidence requires no index writes. A successful snapshot
2526                // never authorizes removal of a tree whose rescue failed.
2527                if let Err(e) = git::snapshot_uncommitted(
2528                    &c.worktree,
2529                    &self
2530                        .state
2531                        .dir()
2532                        .join("artifacts")
2533                        .join(format!("candidate-{label}-recovery")),
2534                )
2535                .await
2536                {
2537                    self.state.event(
2538                        "preserve",
2539                        format!("candidate {label} snapshot incomplete: {e:#}"),
2540                    );
2541                }
2542                let rescued = git::rescue_commit(
2543                    &c.worktree,
2544                    &format!("magi: candidate {label} (uncommitted work)"),
2545                )
2546                .await?;
2547                self.state.note_withheld("implement", &rescued.withheld);
2548                let base = &self.state.base_commit;
2549                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2550                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2551                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2552                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2553                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2554                if c.empty && commits > 0 && !patch.trim().is_empty()
2555                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2556                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2557                        || self.state.tally.is_some())
2558                {
2559                    // Keep the old classification and pending marker so every
2560                    // retry stops too. Retain decisions as recovery evidence;
2561                    // only a supported recovery operation may replace them.
2562                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2563                }
2564                let c = &mut self.state.candidates[i];
2565                c.commits = commits;
2566                c.stat = stat;
2567                c.files = files;
2568                let no_changes = commits == 0 || patch.trim().is_empty();
2569                c.empty = no_changes && !interrupted;
2570                if interrupted && no_changes {
2571                    c.failed = None;
2572                    c.verified_noop = None;
2573                }
2574                if !c.empty {
2575                    c.failed = None;
2576                    c.verified_noop = None;
2577                }
2578                self.state.event(
2579                    "preserve",
2580                    format!("candidate {label}: {files} files, {commits} commits"),
2581                );
2582                Ok::<_, anyhow::Error>(())
2583            }
2584            .await;
2585            match result {
2586                Ok(()) => {
2587                    self.state.candidate_preservation.remove(&label);
2588                    if !interrupted || self.state.candidates[i].files > 0 {
2589                        self.state.interrupted_implementations.remove(&label);
2590                    }
2591                }
2592                Err(e) => {
2593                    let reason = format!("candidate {label} preservation failed: {e:#}");
2594                    self.state.event("preserve", &reason);
2595                    self.state.candidate_preservation.insert(label, reason);
2596                }
2597            }
2598            if self.state.candidate_preservation.is_empty()
2599                && self.state.status == RunStatus::Blocked
2600                && self.state.candidates.iter().all(|c| c.failed.is_none())
2601            {
2602                // Persist the recovered phase with the cleared pending marker;
2603                // a crash between saves must not turn this into a selected-seat
2604                // failure hold on the next resume.
2605                self.state.status = RunStatus::Implementing;
2606                self.state.event(
2607                    "preserve",
2608                    "candidate persistence recovered; implementation can continue",
2609                );
2610            }
2611            self.state.save()?;
2612        }
2613        if !self.state.candidate_preservation.is_empty() {
2614            self.state.status = RunStatus::Blocked;
2615            self.state.save()?;
2616            bail!(
2617                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2618            );
2619        }
2620        Ok(())
2621    }
2622
2623    /// Ask again, once, for work a CLI did and then failed to hand over.
2624    ///
2625    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2626    /// status with an empty response and a usage report showing output tokens,
2627    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2628    /// seven minutes and 14,267 output tokens that arrived as an empty
2629    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2630    ///
2631    /// Two conditions, and both matter:
2632    ///
2633    /// - **Only when the tree is untouched.** Often the agent has already
2634    ///   written its files and only the closing message was lost; the rescue
2635    ///   commit below picks that up and there is nothing to ask for. Re-asking
2636    ///   then would pay for a second implementation of work already on disk.
2637    /// - **Once.** A CLI that drops one stream can drop the next, and this
2638    ///   node is the most expensive in the graph.
2639    ///
2640    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2641    /// dropped reply still carried its `conversation_id`, so the seat is asked
2642    /// to finish what it was doing rather than sent the whole task again. It
2643    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2644    /// node's - for the same reason a re-ranked judge does: restating finished
2645    /// work is not the work.
2646    ///
2647    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2648    /// same way until it resets, while an abandoned conversation is still
2649    /// there to be picked up.
2650    async fn resume_undelivered(
2651        &mut self,
2652        results: &mut [(usize, SeatState, AgentOutcome)],
2653        sent: &[SeatJob],
2654        prompts: &Prompts,
2655        run_id: &str,
2656    ) {
2657        for (wi, seat, out) in results.iter_mut() {
2658            let Some(dropped) = (match &*out {
2659                AgentOutcome::Dropped(o) => o.dropped.clone(),
2660                _ => None,
2661            }) else {
2662                continue;
2663            };
2664            let Some(job) = sent.get(*wi) else { continue };
2665            // Already on disk? Then only the closing message was lost.
2666            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2667                self.state.event(
2668                    "implement",
2669                    format!(
2670                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2671                         work is in the tree",
2672                        seat.key, dropped.output_tokens, dropped.why
2673                    ),
2674                );
2675                continue;
2676            }
2677            // The re-ask only makes sense as a resume: `resume_after_drop`
2678            // says nothing about the task, trusting the seat to still hold it.
2679            // Without a session to resume — sessions disabled, or this CLI's
2680            // drop shape happened not to carry a session id — that prompt
2681            // would open a brand-new conversation with no context at all,
2682            // which is worse than leaving this as the ordinary failure it
2683            // already is.
2684            if !has_context(&job.spec, seat, job.sessions) {
2685                self.state.event(
2686                    "implement",
2687                    format!(
2688                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2689                         is no session left to resume",
2690                        seat.key, dropped.output_tokens, dropped.why
2691                    ),
2692                );
2693                continue;
2694            }
2695            self.state.event(
2696                "implement",
2697                format!(
2698                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2699                     conversation",
2700                    seat.key, dropped.output_tokens, dropped.why
2701                ),
2702            );
2703            let mut retry = job.clone();
2704            retry.seat = seat.clone();
2705            retry.prompt = prompt::resume_after_drop(&dropped.why);
2706            retry.timeout = retry_budget(job.timeout, true);
2707            retry.stem = format!("{}-resume", job.stem);
2708            let cache = self.state.config.cache_dir();
2709            let ctx = WaveCtx {
2710                carry_seats: false,
2711                pauses: Some((&self.pause, &self.interrupt)),
2712                run: run_id,
2713                node: "implement",
2714                prompts,
2715                cache: cache.as_deref(),
2716                round: None,
2717            };
2718            let (resumed_seat, resumed) =
2719                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2720            *seat = resumed_seat;
2721            *out = resumed;
2722        }
2723    }
2724
2725    /// Fall an implement seat through to the next untried agent in the
2726    /// implementer roster when it lost to quota — or, since the handover was
2727    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2728    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2729    /// quota path itself being unchanged — instead of leaving the
2730    /// seat's loss final the moment one agent's account runs dry.
2731    ///
2732    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2733    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2734    /// truncates to the single slot rotation picked, so a solo task whose one
2735    /// implementer hits quota mid-run used to have nothing else to try. This
2736    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2737    /// unrotated roster — which is the only place the *other* candidates in
2738    /// the machine's roster still exist once `implementers` has been cut down
2739    /// to size.
2740    ///
2741    /// Walks forward from just past the seat's own original position in the
2742    /// roster, never wrapping back to the front: a later candidate slot (say
2743    /// `beta`, the roster's second entry) must fall through to the *next*
2744    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2745    /// almost certainly a different candidate's own agent already — and once
2746    /// the roster's tail is exhausted there is nothing left to fall through
2747    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2748    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2749    /// let this retry that id forever. The loop keeps falling through until
2750    /// an attempt lands something other than `Quota` or the roster's tail
2751    /// runs out of untried ids, at which point the seat is left exactly as
2752    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2753    /// `QuotaLoss` recorded, the candidate failed/empty.
2754    ///
2755    /// `sent` is taken mutably and updated with the fallback agent's spec:
2756    /// `resume_unconfirmed_commands`, which runs after this and also reads
2757    /// `sent`, must see whichever agent actually ended up answering the seat
2758    /// — reading the stale, original spec there would check session
2759    /// eligibility against the wrong CLI and could hand a fallback agent's
2760    /// session id to the agent that just lost the seat to quota.
2761    ///
2762    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2763    /// — `self.seat` only reuses state when the agent id is unchanged, so
2764    /// handing it a different id already gets this for free. Reusing the old
2765    /// seat would resume a different CLI's session as if it were a
2766    /// continuation of this one.
2767    ///
2768    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2769    /// a quota loss cuts an agent off mid-turn, so anything already in the
2770    /// tree is unfinished work, not a completed candidate a re-ask would pay
2771    /// for twice. A dirty tree is rescued into a commit first (the same
2772    /// neutral-identity rescue `implement`'s own outcome loop gives every
2773    /// candidate) so the next agent starts clean.
2774    ///
2775    /// The new agent gets the implementer's full prompt and full
2776    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2777    /// it has no session and no context, and is implementing the task from
2778    /// nothing, unlike a resumed drop which is only restating work already
2779    /// done.
2780    ///
2781    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2782    /// `implement` event, never into `self.state.quota` — that is what
2783    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2784    /// go unspent, and a seat that ultimately recovered on its second or
2785    /// third agent is not the stalled panel that check exists to catch. Only
2786    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2787    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2788    /// outcome loop already has — this helper never pushes to it itself.
2789    async fn resume_seat_handovers(
2790        &mut self,
2791        results: &mut [(usize, SeatState, AgentOutcome)],
2792        sent: &mut [SeatJob],
2793        prompts: &Prompts,
2794        run_id: &str,
2795    ) -> Result<()> {
2796        let instruction = seeded_instruction(&self.state);
2797        let language = self.state.config.graph.language.clone();
2798        let github_language = self.state.config.graph.github_language.clone();
2799        let brief = self
2800            .state
2801            .advice
2802            .as_ref()
2803            .and_then(|a| a.synthesis.as_deref())
2804            .map(str::to_owned);
2805        let attachments = self.state.attachments.clone();
2806        for (wi, seat, out) in results.iter_mut() {
2807            // Who holds the other candidate seats of this wave right now
2808            // (earlier handovers already written back to `sent`).
2809            let others: BTreeSet<String> = sent
2810                .iter()
2811                .enumerate()
2812                .filter(|(j, _)| j != wi)
2813                .map(|(_, j)| j.spec.id.clone())
2814                .collect();
2815            let Some(job) = sent.get_mut(*wi) else {
2816                continue;
2817            };
2818            // Where the seat's own original agent sits in the roster — the
2819            // fallback walk starts just past here, never at the front, so a
2820            // later candidate slot's quota loss does not fall back onto an
2821            // earlier slot's own agent.
2822            let start = self
2823                .roles
2824                .implementer_roster
2825                .iter()
2826                .position(|s| s.id == job.spec.id)
2827                .unwrap_or(0);
2828            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2829            let mut fallback_attempt = 0usize;
2830            let mut prev: Option<FailClass> = None;
2831            while let Some(cur) = FailClass::of(&*out) {
2832                if !should_hand_over(prev.as_ref(), &cur) {
2833                    break;
2834                }
2835                let Some(next) =
2836                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2837                        .cloned()
2838                else {
2839                    break;
2840                };
2841                tried.insert(next.id.clone());
2842                fallback_attempt += 1;
2843
2844                let coordination = self.coordinate(&job.cwd).await?;
2845                if let Ok(r) = git::rescue_commit(
2846                    &job.cwd,
2847                    &format!(
2848                        "magi: candidate {} (uncommitted work before {} fallback)",
2849                        seat.key,
2850                        if cur == FailClass::Quota {
2851                            "quota"
2852                        } else {
2853                            "handover"
2854                        }
2855                    ),
2856                )
2857                .await
2858                {
2859                    self.state.note_withheld("implement", &r.withheld);
2860                }
2861                drop(coordination);
2862
2863                record_handover(
2864                    &mut self.state,
2865                    "implement",
2866                    &seat.key,
2867                    &seat.agent,
2868                    &next.id,
2869                    &cur,
2870                    &fail_reason(&*out),
2871                );
2872                prev = Some(cur.clone());
2873
2874                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2875                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2876                // Kept in sync on `sent` itself, not just the local retry: a
2877                // later helper (`resume_unconfirmed_commands`) reads `sent`
2878                // after this one returns and must see whichever agent is now
2879                // occupying the seat, not the one that just quota'd out —
2880                // otherwise it would judge session/continuation eligibility
2881                // by the wrong CLI and could resend a fallback's session id
2882                // to the agent that lost it the seat in the first place.
2883                job.spec = next.clone();
2884                let mut retry = job.clone();
2885                retry.seat = new_seat;
2886                retry.prompt = prompt::implement_in(
2887                    &instruction,
2888                    &job.cwd.to_string_lossy(),
2889                    &language,
2890                    &github_language,
2891                    brief.as_deref(),
2892                    &attachments,
2893                );
2894                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2895                let cache = self.state.config.cache_dir();
2896                let ctx = WaveCtx {
2897                    carry_seats: false,
2898                    pauses: Some((&self.pause, &self.interrupt)),
2899                    run: run_id,
2900                    node: "implement",
2901                    prompts,
2902                    cache: cache.as_deref(),
2903                    round: None,
2904                };
2905                let (fallback_seat, fallback_out) = run_one(
2906                    retry,
2907                    Arc::clone(&self.sem),
2908                    &ctx,
2909                    &mut self.state,
2910                    fallback_attempt,
2911                )
2912                .await;
2913                *seat = fallback_seat;
2914                *out = fallback_out;
2915            }
2916        }
2917        Ok(())
2918    }
2919
2920    /// Ask an implement seat's own CLI to confirm what it started, once, when
2921    /// its reply reported a command whose completion status it never
2922    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2923    /// that does and does not mean.
2924    ///
2925    /// The completion contract this task asks for, extended to `implement`
2926    /// with the same signal `continue_fix_report` reads for the fixer,
2927    /// rather than a keyword search over the reply or a hard requirement on
2928    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2929    /// a candidate's CLI turn ended cleanly while a test run it had started
2930    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2931    /// named in it at all is untouched by this: `commands` is empty, so
2932    /// there is nothing to be unconfirmed.
2933    ///
2934    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2935    /// this is not about recovering edits that might already be on disk, it
2936    /// is about a result the seat itself never vouched for, which resuming
2937    /// asks for regardless of what the tree already holds. Bounded to one
2938    /// attempt for the same reason `resume_undelivered` is — this is the
2939    /// most expensive node in the graph — and a seat that still cannot
2940    /// confirm on that attempt is left as whatever its (possibly still
2941    /// unconfirmed) reply says; this does not invent a new "failed" reason
2942    /// for a candidate that otherwise produced a real, committed change.
2943    async fn resume_unconfirmed_commands(
2944        &mut self,
2945        results: &mut [(usize, SeatState, AgentOutcome)],
2946        sent: &[SeatJob],
2947        prompts: &Prompts,
2948        run_id: &str,
2949    ) {
2950        for (wi, seat, out) in results.iter_mut() {
2951            let AgentOutcome::Ok(o) = &*out else {
2952                continue;
2953            };
2954            if !has_unconfirmed_command(&o.commands) {
2955                continue;
2956            }
2957            let Some(job) = sent.get(*wi) else { continue };
2958            if !has_context(&job.spec, seat, job.sessions) {
2959                self.state.event(
2960                    "implement",
2961                    format!(
2962                        "{}: the reply named a command whose own CLI never confirmed the exit \
2963                         status of, but there is no session left to resume",
2964                        seat.key
2965                    ),
2966                );
2967                continue;
2968            }
2969            self.state.event(
2970                "implement",
2971                format!(
2972                    "{}: the reply named a command whose own CLI never confirmed the exit \
2973                     status of; resuming the conversation",
2974                    seat.key
2975                ),
2976            );
2977            let mut retry = job.clone();
2978            retry.seat = seat.clone();
2979            retry.prompt = prompt::resume_incomplete(
2980                "a command in your last reply had no confirmed exit status",
2981            );
2982            retry.timeout = retry_budget(job.timeout, true);
2983            retry.stem = format!("{}-confirm", job.stem);
2984            let cache = self.state.config.cache_dir();
2985            let ctx = WaveCtx {
2986                carry_seats: false,
2987                pauses: Some((&self.pause, &self.interrupt)),
2988                run: run_id,
2989                node: "implement",
2990                prompts,
2991                cache: cache.as_deref(),
2992                round: None,
2993            };
2994            let (resumed_seat, resumed) =
2995                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2996            *seat = resumed_seat;
2997            *out = resumed;
2998        }
2999    }
3000
3001    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3002    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3003    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3004    /// that motivated this.
3005    ///
3006    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3007    /// own nudge loop already covers for judge/review/vote seats, and not a
3008    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3009    /// implement seats: here the CLI turn genuinely finished while the node's
3010    /// own work — the fixer's account of what it did — had not. Gated purely
3011    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3012    /// reply, never on any wording in it, so a fixer whose valid, first-try
3013    /// `FixReport` happens to mention having waited on a background test is
3014    /// never resumed — the `Ok(report)` branch at the call site returns
3015    /// before this is ever invoked.
3016    ///
3017    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3018    /// attempt ([`retry_budget`]), nothing attempted once the session is
3019    /// gone, and a quota hit ends the loop immediately rather than retrying a
3020    /// rate limit that fails the same way again.
3021    async fn continue_fix_report(
3022        &mut self,
3023        mut seat: SeatState,
3024        parse_err: String,
3025        job: &SeatJob,
3026        prompts: &Prompts,
3027        run_id: &str,
3028        round: usize,
3029    ) -> (
3030        SeatState,
3031        Option<FixReport>,
3032        Option<String>,
3033        ContinuationRecord,
3034    ) {
3035        let mut last_err = parse_err;
3036        let mut cumulative_wait_ms = 0u64;
3037        let mut attempts = 0usize;
3038        loop {
3039            if !has_context(&job.spec, &seat, job.sessions) {
3040                self.state.event(
3041                    "fix",
3042                    format!(
3043                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3044                         session left to resume into"
3045                    ),
3046                );
3047                let outcome = if attempts == 0 {
3048                    ContinuationOutcome::NoSession
3049                } else {
3050                    ContinuationOutcome::Exhausted
3051                };
3052                return (
3053                    seat,
3054                    None,
3055                    Some(format!("unparsable fix report: {last_err}")),
3056                    ContinuationRecord {
3057                        attempts,
3058                        cumulative_wait_ms,
3059                        outcome,
3060                    },
3061                );
3062            }
3063            if attempts >= MAX_FIX_CONTINUATIONS {
3064                self.state.event(
3065                    "fix",
3066                    format!(
3067                        "round {round}: fixer's reply still had no adoption report after \
3068                         {attempts} continuation(s) ({last_err}); giving up"
3069                    ),
3070                );
3071                return (
3072                    seat,
3073                    None,
3074                    Some(format!(
3075                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3076                    )),
3077                    ContinuationRecord {
3078                        attempts,
3079                        cumulative_wait_ms,
3080                        outcome: ContinuationOutcome::Exhausted,
3081                    },
3082                );
3083            }
3084            attempts += 1;
3085            self.state.event(
3086                "fix",
3087                format!(
3088                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3089                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3090                ),
3091            );
3092            let mut retry = job.clone();
3093            retry.seat = seat.clone();
3094            retry.prompt = prompt::resume_incomplete(&last_err);
3095            retry.timeout = retry_budget(job.timeout, true);
3096            retry.stem = format!("{}-continue{attempts}", job.stem);
3097            let cache = self.state.config.cache_dir();
3098            let ctx = WaveCtx {
3099                carry_seats: false,
3100                pauses: Some((&self.pause, &self.interrupt)),
3101                run: run_id,
3102                node: "fix",
3103                prompts,
3104                cache: cache.as_deref(),
3105                round: Some(round),
3106            };
3107            let (resumed_seat, resumed_out) = run_one(
3108                retry,
3109                Arc::clone(&self.sem),
3110                &ctx,
3111                &mut self.state,
3112                attempts,
3113            )
3114            .await;
3115            seat = resumed_seat;
3116            match resumed_out {
3117                AgentOutcome::Ok(o) => {
3118                    cumulative_wait_ms += o.duration_ms;
3119                    match verdict::extract_json::<FixReport>(&o.text) {
3120                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3121                            self.state.event(
3122                                "fix",
3123                                format!(
3124                                    "round {round}: fixer's adoption report recovered after \
3125                                     {attempts} continuation(s)"
3126                                ),
3127                            );
3128                            return (
3129                                seat,
3130                                Some(report),
3131                                None,
3132                                ContinuationRecord {
3133                                    attempts,
3134                                    cumulative_wait_ms,
3135                                    outcome: ContinuationOutcome::Resumed,
3136                                },
3137                            );
3138                        }
3139                        // The report parsed, but this same reply's own
3140                        // CommandEvidence — the identical record `state.jobs`
3141                        // renders — names a command whose CLI never
3142                        // confirmed an exit status. Read together, that is
3143                        // not a resolved answer: keep nudging rather than
3144                        // accept a report standing next to a command the
3145                        // seat's own CLI cannot vouch for.
3146                        Ok(_) => {
3147                            last_err = "the reply parsed, but it reported a command whose own CLI \
3148                                 never confirmed an exit status"
3149                                .to_owned();
3150                        }
3151                        Err(e) => last_err = e.to_string(),
3152                    }
3153                }
3154                AgentOutcome::Quota(o) => {
3155                    cumulative_wait_ms += o.duration_ms;
3156                    self.state.quota.push(QuotaLoss {
3157                        seat: seat.key.clone(),
3158                        node: "fix".to_owned(),
3159                        at: Timestamp::now(),
3160                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3161                    });
3162                    self.state.event(
3163                        "fix",
3164                        format!(
3165                            "round {round}: continuation rate limited (quota); not retrying now"
3166                        ),
3167                    );
3168                    return (
3169                        seat,
3170                        None,
3171                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3172                        ContinuationRecord {
3173                            attempts,
3174                            cumulative_wait_ms,
3175                            outcome: ContinuationOutcome::QuotaLost,
3176                        },
3177                    );
3178                }
3179                AgentOutcome::Dropped(o) => {
3180                    cumulative_wait_ms += o.duration_ms;
3181                    let why = o
3182                        .dropped
3183                        .as_ref()
3184                        .map(|d| d.why.as_str())
3185                        .unwrap_or("the CLI ended the stream without delivering its answer");
3186                    last_err = format!("the CLI dropped the stream ({why})");
3187                }
3188                AgentOutcome::Failed(e) => last_err = e,
3189                // The continuation was never started; the report stays lost
3190                // and the caller records that, without any handover.
3191                AgentOutcome::ResourceBlocked(e) => {
3192                    return (
3193                        seat,
3194                        None,
3195                        Some(format!("the continuation was not started: {e}")),
3196                        ContinuationRecord {
3197                            attempts: attempts.saturating_sub(1),
3198                            cumulative_wait_ms,
3199                            outcome: ContinuationOutcome::NoSession,
3200                        },
3201                    );
3202                }
3203            }
3204        }
3205    }
3206
3207    fn after_implement(&mut self) -> Result<()> {
3208        if !self.state.candidate_preservation.is_empty() {
3209            bail!("candidate preservation incomplete; judging is unsafe");
3210        }
3211        // Scan every candidate patch once the set is complete.
3212        if self.state.leaks.is_empty() {
3213            let cfg = self.state.config.blind.clone();
3214            let mut leaks = Vec::new();
3215            for c in &self.state.candidates {
3216                let Some(patch) =
3217                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3218                else {
3219                    continue;
3220                };
3221                leaks.extend(blind::scan(
3222                    &format!("candidate {} patch", c.label),
3223                    &patch,
3224                    &cfg.vendor_tokens,
3225                ));
3226            }
3227            if !leaks.is_empty() {
3228                let summary = leaks
3229                    .iter()
3230                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3231                    .collect::<Vec<_>>()
3232                    .join(", ");
3233                match cfg.on_leak {
3234                    LeakPolicy::Fail => {
3235                        self.state.status = RunStatus::Failed;
3236                        self.state
3237                            .event("blind", format!("vendor text in a patch: {summary}"));
3238                        self.state.leaks = leaks;
3239                        self.state.save()?;
3240                        self.settle_questions();
3241                        bail!(
3242                            "blind.on_leak = \"fail\" and vendor text reached a \
3243                             judged patch: {summary}"
3244                        );
3245                    }
3246                    LeakPolicy::Redact => self.state.event(
3247                        "blind",
3248                        format!("redacting vendor text for judging: {summary}"),
3249                    ),
3250                    LeakPolicy::Warn => self.state.event(
3251                        "blind",
3252                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3253                    ),
3254                }
3255                self.state.leaks = leaks;
3256            }
3257        }
3258
3259        if self.state.viable().is_empty() {
3260            if self.state.all_candidates_verified_noop() {
3261                // Every candidate agreed, with evidence the adoption guard
3262                // accepted, that nothing belongs in this worktree. That is
3263                // not the same fact as a candidate that simply failed to
3264                // write anything, and settling it as an ordinary `Failed`
3265                // (see `SCHEMA`'s doc for schema 10) is what let two of
3266                // task 391f's attempts burn a retry each re-discovering the
3267                // same already-landed fix. Terminal either way, so `judge`
3268                // must never run over an empty candidate set — unlike the
3269                // `Failed` branch below this returns `Ok`, not an error:
3270                // nothing here failed.
3271                self.state.status = RunStatus::VerifiedNoop;
3272                self.state.save()?;
3273                self.settle_questions();
3274                return Ok(());
3275            }
3276            self.state.status = RunStatus::Failed;
3277            self.state.save()?;
3278            self.settle_questions();
3279            bail!("no candidate produced a change; nothing to judge");
3280        }
3281        self.state.status = RunStatus::Judging;
3282        self.state.save()?;
3283        Ok(())
3284    }
3285
3286    // --------------------------------------------------------------- judge
3287
3288    async fn judge(&mut self) -> Result<()> {
3289        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3290        // agent files with `magi task add` name the run that paid for it. The
3291        // prompt overlay is cloned alongside it because the waves borrow it
3292        // while `self` is mutably borrowed by the node's own bookkeeping.
3293        let run_id = self.state.id.clone();
3294        let prompts = self.state.config.prompts.clone();
3295        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3296            return Ok(());
3297        }
3298        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3299        if viable.len() == 1 {
3300            // Recorded so this is a one-time event: `judgements` stays empty
3301            // either way, which without this flag is indistinguishable from
3302            // "not yet judged" on the next reentry — and status is left
3303            // untouched, so a later node's conclusion (e.g. `Blocked` after
3304            // the review budget ran out) survives a resume instead of being
3305            // clobbered back to `Judging` by this node running again.
3306            self.state.judge_skipped = true;
3307            self.state.event(
3308                "judge",
3309                format!(
3310                    "only candidate {} produced a change; judging skipped",
3311                    viable[0].label
3312                ),
3313            );
3314            self.state.save()?;
3315            return Ok(());
3316        }
3317        self.state.status = RunStatus::Judging;
3318
3319        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3320        let language = self.state.config.graph.language.clone();
3321        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3322        let sessions = self.state.config.graph.sessions;
3323        let artifacts = agent::artifacts_dir(&self.state.dir());
3324        let root = self.state.worktree_root();
3325        let base_short = short(&self.state.base_commit);
3326
3327        let mut jobs = Vec::new();
3328        let mut orders = Vec::new();
3329        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3330            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3331            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3332            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3333            let seat_key = format!("judge-{}", j + 1);
3334            let seat = self.seat(&seat_key, &spec.id);
3335            jobs.push(SeatJob {
3336                prompt: prompt::judge(
3337                    &self.state.instruction,
3338                    &views,
3339                    self.roles.judges.len(),
3340                    &base_short,
3341                    &language,
3342                ),
3343                spec,
3344                seat,
3345                cwd: root.join(format!("judge-{}", j + 1)),
3346                timeout,
3347                allow_write: false,
3348                sessions,
3349                artifacts: artifacts.clone(),
3350                stem: format!("judge-{}", j + 1),
3351                handover: None,
3352            });
3353        }
3354
3355        self.state.event(
3356            "judge",
3357            format!(
3358                "{} judges ranking {} candidates blind",
3359                jobs.len(),
3360                viable.len()
3361            ),
3362        );
3363        let labels_for_check = labels.clone();
3364        let mut quota_losses = Vec::new();
3365        let cache = self.state.config.cache_dir();
3366        let ctx = WaveCtx {
3367            carry_seats: false,
3368            pauses: Some((&self.pause, &self.interrupt)),
3369            run: &run_id,
3370            node: "judge",
3371            prompts: &prompts,
3372            cache: cache.as_deref(),
3373            round: None,
3374        };
3375        let results = ask_json_wave::<Ranking>(
3376            jobs,
3377            Arc::clone(&self.sem),
3378            self.state.config.graph.retries,
3379            &self.roles.judge_roster,
3380            &ctx,
3381            &mut quota_losses,
3382            &mut self.state,
3383            &move |r: &Ranking| r.validate(&labels_for_check),
3384        )
3385        .await;
3386        if self.state.resource_block.is_some() {
3387            return self.stop_for_resource();
3388        }
3389        self.state.quota.extend(quota_losses);
3390
3391        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3392            let agent_id = seat.agent.clone();
3393            self.state.seats.insert(seat.key.clone(), seat);
3394            let mut record = Judgement {
3395                judge: j + 1,
3396                seat: format!("judge-{}", j + 1),
3397                agent: agent_id,
3398                ranking: Vec::new(),
3399                reasons: BTreeMap::new(),
3400                confidence: None,
3401                strengths: BTreeMap::new(),
3402                order: orders[j].clone(),
3403                failed: None,
3404                duration_ms: 0,
3405            };
3406            match res {
3407                Ok((ranking, out)) => {
3408                    record.ranking = ranking.normalized();
3409                    record.reasons = ranking.reasons;
3410                    record.strengths = ranking.strengths;
3411                    record.confidence = ranking.confidence;
3412                    record.duration_ms = out.duration_ms;
3413                    self.state.event(
3414                        "judge",
3415                        format!(
3416                            "judge {} ranked {}",
3417                            j + 1,
3418                            record.ranking.iter().collect::<String>()
3419                        ),
3420                    );
3421                }
3422                Err(e) => {
3423                    record.failed = Some(e.to_string());
3424                    self.state
3425                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3426                }
3427            }
3428            self.state.judgements.push(record);
3429            self.state.save()?;
3430        }
3431        Ok(())
3432    }
3433
3434    // ---------------------------------------------------------- deliberate
3435
3436    async fn deliberate(&mut self) -> Result<()> {
3437        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3438        // agent files with `magi task add` name the run that paid for it. The
3439        // prompt overlay is cloned alongside it because the waves borrow it
3440        // while `self` is mutably borrowed by the node's own bookkeeping.
3441        let run_id = self.state.id.clone();
3442        let prompts = self.state.config.prompts.clone();
3443        if !self.state.deliberation.is_empty() {
3444            return Ok(());
3445        }
3446        let tops: Vec<char> = self
3447            .state
3448            .judgements
3449            .iter()
3450            .filter_map(|j| j.ranking.first().copied())
3451            .collect();
3452        let rounds = self.state.config.graph.deliberate_rounds;
3453        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3454            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3455                self.state.event(
3456                    "deliberate",
3457                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3458                );
3459            }
3460            self.state.status = RunStatus::Voting;
3461            self.state.save()?;
3462            return Ok(());
3463        }
3464
3465        self.state.status = RunStatus::Deliberating;
3466        self.state.event(
3467            "deliberate",
3468            format!(
3469                "split: first choices were {} — opening {rounds} round(s)",
3470                tops.iter().collect::<String>()
3471            ),
3472        );
3473
3474        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3475        let language = self.state.config.graph.language.clone();
3476        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3477        let sessions = self.state.config.graph.sessions;
3478        let artifacts = agent::artifacts_dir(&self.state.dir());
3479        let root = self.state.worktree_root();
3480        let base_short = short(&self.state.base_commit);
3481
3482        // Judges argue in sequence so that a turn can answer the one before it;
3483        // that is the difference between deliberation and three parallel
3484        // monologues.
3485        for round in 1..=rounds {
3486            let mut turns: Vec<DeliberationTurn> = Vec::new();
3487            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3488                if self.state.judgements[j].failed.is_some() {
3489                    continue;
3490                }
3491                let seat_key = format!("judge-{}", j + 1);
3492                let spec = self.occupant(&seat_key, spec);
3493                let mut seat = self.seat(&seat_key, &spec.id);
3494                let transcript = self.transcript(&turns, j);
3495                let build = |context: Option<&str>| {
3496                    prompt::deliberate(
3497                        &self.state.instruction,
3498                        context,
3499                        &transcript,
3500                        round,
3501                        rounds,
3502                        &language,
3503                    )
3504                };
3505                let block = self.candidate_block(&viable, &base_short);
3506                let full = build(Some(&block));
3507                let text = if has_context(&spec, &seat, sessions) {
3508                    build(None)
3509                } else {
3510                    full.clone()
3511                };
3512                let job = SeatJob {
3513                    spec,
3514                    seat: seat.clone(),
3515                    prompt: text,
3516                    cwd: root.join(format!("judge-{}", j + 1)),
3517                    timeout,
3518                    allow_write: false,
3519                    sessions,
3520                    artifacts: artifacts.clone(),
3521                    stem: format!("delib-{round}-judge-{}", j + 1),
3522                    handover: Some(full),
3523                };
3524                let cache = self.state.config.cache_dir();
3525                let ctx = WaveCtx {
3526                    carry_seats: false,
3527                    pauses: Some((&self.pause, &self.interrupt)),
3528                    run: &run_id,
3529                    node: "deliberate",
3530                    prompts: &prompts,
3531                    cache: cache.as_deref(),
3532                    round: None,
3533                };
3534                // A turn is never nudged (`retries` 0): a failed seat is
3535                // handed to the next roster agent, which gets the full
3536                // context. An empty answer is a turn, not a failure.
3537                let mut losses = Vec::new();
3538                let mut results = ask_wave_with::<String>(
3539                    vec![job],
3540                    Arc::clone(&self.sem),
3541                    0,
3542                    &self.roles.judge_roster,
3543                    &ctx,
3544                    &mut losses,
3545                    &mut self.state,
3546                    &|text: &str| {
3547                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3548                    },
3549                )
3550                .await;
3551                self.state.quota.extend(losses);
3552                let (updated, res, _) = results.pop().expect("one job in, one result out");
3553                seat = updated;
3554                let agent_id = seat.agent.clone();
3555                self.state.seats.insert(seat.key.clone(), seat);
3556                let body = match res {
3557                    Ok((body, _)) => body,
3558                    // Skip the seat; a CLI's raw error JSON is never read as
3559                    // this judge's position.
3560                    Err(e) => {
3561                        self.state
3562                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3563                        continue;
3564                    }
3565                };
3566                let tentative = verdict::extract_json::<Position>(&body)
3567                    .ok()
3568                    .and_then(|p| p.tentative)
3569                    .and_then(|s| s.trim().chars().next())
3570                    .map(|c| c.to_ascii_uppercase());
3571                self.state.event(
3572                    "deliberate",
3573                    format!(
3574                        "round {round}: judge {} now favours {}",
3575                        j + 1,
3576                        tentative.map_or("—".to_owned(), |c| c.to_string())
3577                    ),
3578                );
3579                turns.push(DeliberationTurn {
3580                    judge: j + 1,
3581                    agent: agent_id,
3582                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3583                    tentative,
3584                });
3585            }
3586            self.state
3587                .deliberation
3588                .push(DeliberationRound { round, turns });
3589            self.state.save()?;
3590        }
3591
3592        self.state.status = RunStatus::Voting;
3593        self.state.save()?;
3594        Ok(())
3595    }
3596
3597    // ---------------------------------------------------------------- vote
3598
3599    async fn vote(&mut self) -> Result<()> {
3600        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3601        // agent files with `magi task add` name the run that paid for it. The
3602        // prompt overlay is cloned alongside it because the waves borrow it
3603        // while `self` is mutably borrowed by the node's own bookkeeping.
3604        let run_id = self.state.id.clone();
3605        let prompts = self.state.config.prompts.clone();
3606        if !self.state.votes.is_empty() {
3607            return Ok(());
3608        }
3609        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3610        if viable.len() == 1 {
3611            return Ok(());
3612        }
3613        self.state.status = RunStatus::Voting;
3614
3615        let language = self.state.config.graph.language.clone();
3616        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3617        let sessions = self.state.config.graph.sessions;
3618        let artifacts = agent::artifacts_dir(&self.state.dir());
3619        let root = self.state.worktree_root();
3620        let base_short = short(&self.state.base_commit);
3621        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3622
3623        let mut jobs = Vec::new();
3624        let mut seats_at = Vec::new();
3625        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3626            if self
3627                .state
3628                .judgements
3629                .get(j)
3630                .is_some_and(|r| r.failed.is_some())
3631            {
3632                continue;
3633            }
3634            let seat_key = format!("judge-{}", j + 1);
3635            let spec = self.occupant(&seat_key, spec);
3636            let seat = self.seat(&seat_key, &spec.id);
3637            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3638            let text = if has_context(&spec, &seat, sessions) {
3639                prompt::final_vote(&viable, &language)
3640            } else {
3641                full.clone()
3642            };
3643            jobs.push(SeatJob {
3644                spec,
3645                seat,
3646                prompt: text,
3647                cwd: root.join(format!("judge-{}", j + 1)),
3648                timeout,
3649                allow_write: false,
3650                sessions,
3651                artifacts: artifacts.clone(),
3652                stem: format!("vote-judge-{}", j + 1),
3653                handover: Some(full),
3654            });
3655            seats_at.push(j);
3656        }
3657
3658        self.state.event(
3659            "vote",
3660            format!(
3661                "collecting {} final votes one by one, privately",
3662                jobs.len()
3663            ),
3664        );
3665        let allowed = viable.clone();
3666        let mut quota_losses = Vec::new();
3667        let cache = self.state.config.cache_dir();
3668        let ctx = WaveCtx {
3669            carry_seats: false,
3670            pauses: Some((&self.pause, &self.interrupt)),
3671            run: &run_id,
3672            node: "vote",
3673            prompts: &prompts,
3674            cache: cache.as_deref(),
3675            round: None,
3676        };
3677        let results = ask_json_wave::<FinalVote>(
3678            jobs,
3679            Arc::clone(&self.sem),
3680            self.state.config.graph.retries,
3681            &self.roles.judge_roster,
3682            &ctx,
3683            &mut quota_losses,
3684            &mut self.state,
3685            &move |v: &FinalVote| match v.label() {
3686                Some(c) if allowed.contains(&c) => Ok(()),
3687                other => bail!("vote {other:?} is not one of {allowed:?}"),
3688            },
3689        )
3690        .await;
3691        if self.state.resource_block.is_some() {
3692            return self.stop_for_resource();
3693        }
3694        self.state.quota.extend(quota_losses);
3695
3696        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3697            let agent_id = seat.agent.clone();
3698            self.state.seats.insert(seat.key.clone(), seat);
3699            let initial = self
3700                .state
3701                .judgements
3702                .get(j)
3703                .and_then(|r| r.ranking.first().copied());
3704            let mut record = VoteRecord {
3705                judge: j + 1,
3706                agent: agent_id,
3707                vote: None,
3708                reason: String::new(),
3709                changed: false,
3710            };
3711            match res {
3712                Ok((v, _)) => {
3713                    record.vote = v.label();
3714                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3715                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3716                    self.state.event(
3717                        "vote",
3718                        format!(
3719                            "judge {} voted {}{}",
3720                            j + 1,
3721                            record.vote.unwrap_or('?'),
3722                            if record.changed { " (changed)" } else { "" }
3723                        ),
3724                    );
3725                }
3726                Err(e) => {
3727                    self.state
3728                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3729                }
3730            }
3731            self.state.votes.push(record);
3732            self.state.save()?;
3733        }
3734        Ok(())
3735    }
3736
3737    // --------------------------------------------------------------- tally
3738
3739    fn tally(&mut self) -> Result<()> {
3740        if self.state.tally.is_some() {
3741            return Ok(());
3742        }
3743        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3744        let tops: Vec<char> = self
3745            .state
3746            .judgements
3747            .iter()
3748            .filter_map(|j| j.ranking.first().copied())
3749            .collect();
3750        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3751
3752        // A judge whose private vote failed still counted once, in the initial
3753        // ranking; using it beats discarding a whole seat.
3754        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3755        let mut cast: Vec<char> = Vec::new();
3756        for (i, j) in self.state.judgements.iter().enumerate() {
3757            let vote = self
3758                .state
3759                .votes
3760                .iter()
3761                .find(|v| v.judge == i + 1)
3762                .and_then(|v| v.vote)
3763                .or_else(|| j.ranking.first().copied());
3764            if let Some(v) = vote {
3765                *first_choice.entry(v).or_insert(0) += 1;
3766                cast.push(v);
3767            }
3768        }
3769
3770        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3771        for j in &self.state.judgements {
3772            let n = j.ranking.len();
3773            for (pos, label) in j.ranking.iter().enumerate() {
3774                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3775            }
3776        }
3777
3778        let best = first_choice.values().copied().max().unwrap_or(0);
3779        let mut leaders: Vec<char> = first_choice
3780            .iter()
3781            .filter(|(_, v)| **v == best)
3782            .map(|(k, _)| *k)
3783            .collect();
3784        let mut tie_break = None;
3785        if leaders.len() > 1 {
3786            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3787            let borda_leaders: Vec<char> = leaders
3788                .iter()
3789                .copied()
3790                .filter(|l| borda[l] == top_borda)
3791                .collect();
3792            tie_break = Some(if borda_leaders.len() == 1 {
3793                format!(
3794                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3795                    leaders.len()
3796                )
3797            } else {
3798                format!(
3799                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3800                    leaders.len()
3801                )
3802            });
3803            leaders = borda_leaders;
3804            leaders.sort_unstable();
3805        }
3806        let winner = *leaders
3807            .first()
3808            .or(viable.first())
3809            .context("no candidate to declare a winner from")?;
3810
3811        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3812        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3813        let deliberated = !self.state.deliberation.is_empty();
3814
3815        // Whose verdict is this? A rate-limited seat is absent even if it
3816        // ranked before the limit hit, so presence is measured against the
3817        // recorded losses, not just "did a ranking ever appear".
3818        let quota_seats: std::collections::BTreeSet<&str> =
3819            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3820        let mut present = 0usize;
3821        for (i, j) in self.state.judgements.iter().enumerate() {
3822            if quota_seats.contains(j.seat.as_str()) {
3823                continue;
3824            }
3825            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3826            let voted = self
3827                .state
3828                .votes
3829                .iter()
3830                .any(|v| v.judge == i + 1 && v.vote.is_some());
3831            if ranked || voted {
3832                present += 1;
3833            }
3834        }
3835        // Strict majority of the configured panel. A bare majority is real
3836        // signal we can act on, while a minority verdict must never stand in
3837        // for a healthy one. A one-candidate run needs no panel at all, and
3838        // `judges` stays `0` rather than the roster size a panel that never
3839        // sat would otherwise be credited with.
3840        let needs_quorum = viable.len() > 1;
3841        let judges_total = if needs_quorum {
3842            self.roles.judges.len()
3843        } else {
3844            0
3845        };
3846        let quorum = if needs_quorum {
3847            judges_total / 2 + 1
3848        } else {
3849            0
3850        };
3851        let met_quorum = !needs_quorum || present >= quorum;
3852        let uncontested = (!needs_quorum).then(|| {
3853            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3854        });
3855
3856        self.state.event(
3857            "tally",
3858            match &uncontested {
3859                Some(reason) => format!("winner {winner} — {reason}"),
3860                None => format!(
3861                    "winner {winner} — votes {} | initial {} | {} changed | \
3862                     {present}/{judges_total} judges{}",
3863                    first_choice
3864                        .iter()
3865                        .map(|(k, v)| format!("{k}:{v}"))
3866                        .collect::<Vec<_>>()
3867                        .join(" "),
3868                    if unanimous_initial {
3869                        "unanimous"
3870                    } else {
3871                        "split"
3872                    },
3873                    changed_votes,
3874                    if met_quorum {
3875                        String::new()
3876                    } else {
3877                        format!(" — below quorum ({quorum} required)")
3878                    },
3879                ),
3880            },
3881        );
3882        if !met_quorum {
3883            self.state.event(
3884                "stall",
3885                format!(
3886                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3887                     the run stops here, resumable"
3888                ),
3889            );
3890        }
3891        self.state.tally = Some(Tally {
3892            first_choice,
3893            borda,
3894            winner,
3895            rankings: tops.len(),
3896            unanimous_initial,
3897            deliberated,
3898            changed_votes,
3899            unanimous_final,
3900            tie_break,
3901            judges: judges_total,
3902            present,
3903            quorum,
3904            met_quorum,
3905            uncontested,
3906        });
3907        self.state.status = if met_quorum {
3908            RunStatus::Reviewing
3909        } else {
3910            RunStatus::Stalled
3911        };
3912        self.state.save()?;
3913        Ok(())
3914    }
3915
3916    // ------------------------------------------------------------- recover
3917
3918    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3919    /// resumed toward completion once the transient cause clears.
3920    ///
3921    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3922    /// it toward the quorum, which is exactly the set of seats whose absence
3923    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3924    /// not depend on which node happened to hit the limit), or an ordinary
3925    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3926    /// seat is never disturbed.
3927    ///
3928    /// A seat that now answers with a usable ranking is "recovered": its
3929    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3930    /// `tally` counts it present again), and its vote re-collected. A seat that
3931    /// still fails keeps its loss and stays absent.
3932    ///
3933    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3934    /// to review/gate/merge), `false` when it is still below quorum (the run
3935    /// stays `Stalled`, still resumable for a later retry).
3936    #[allow(clippy::too_many_lines)]
3937    async fn recover_stall(&mut self) -> Result<bool> {
3938        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3939        // agent files with `magi task add` name the run that paid for it. The
3940        // prompt overlay is cloned alongside it because the waves borrow it
3941        // while `self` is mutably borrowed by the node's own bookkeeping.
3942        let run_id = self.state.id.clone();
3943        let prompts = self.state.config.prompts.clone();
3944        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3945        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3946        // is what keeps a non-quota collapse — or a quota loss recorded at the
3947        // deliberate node — from being a permanent dead-end on `--resume`.
3948        let quota_seats: BTreeSet<&str> =
3949            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3950        let absent: Vec<String> = self
3951            .state
3952            .judgements
3953            .iter()
3954            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3955            .map(|j| j.seat.clone())
3956            .collect();
3957        if absent.is_empty() {
3958            return Ok(false);
3959        }
3960        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3961        if viable.len() <= 1 {
3962            return Ok(false);
3963        }
3964        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3965        let language = self.state.config.graph.language.clone();
3966        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3967        let sessions = self.state.config.graph.sessions;
3968        let artifacts = agent::artifacts_dir(&self.state.dir());
3969        let root = self.state.worktree_root();
3970        let base_short = short(&self.state.base_commit);
3971        let candidates: Vec<Candidate> = viable.clone();
3972
3973        // Map each absent seat key to its 0-based position in `roles.judges`.
3974        let mut positions: Vec<usize> = absent
3975            .iter()
3976            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3977            .collect();
3978        if positions.is_empty() {
3979            return Ok(false);
3980        }
3981        positions.sort_unstable();
3982        positions.dedup();
3983
3984        // Re-rank the lost seats, one blind prompt each.
3985        let mut judge_jobs = Vec::new();
3986        for &j in &positions {
3987            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3988            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3989            let seat_key = format!("judge-{}", j + 1);
3990            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3991            let seat = self.seat(&seat_key, &spec.id);
3992            judge_jobs.push(SeatJob {
3993                spec,
3994                seat,
3995                prompt: prompt::judge(
3996                    &self.state.instruction,
3997                    &views,
3998                    self.roles.judges.len(),
3999                    &base_short,
4000                    &language,
4001                ),
4002                cwd: root.join(seat_key),
4003                timeout,
4004                allow_write: false,
4005                sessions,
4006                artifacts: artifacts.clone(),
4007                stem: format!("judge-{}-recover", j + 1),
4008                handover: None,
4009            });
4010        }
4011
4012        let labels_for_check = labels.clone();
4013        let mut judge_losses = Vec::new();
4014        let retries = self.state.config.graph.retries;
4015        let cache = self.state.config.cache_dir();
4016        let ctx = WaveCtx {
4017            carry_seats: false,
4018            pauses: Some((&self.pause, &self.interrupt)),
4019            run: &run_id,
4020            node: "judge",
4021            prompts: &prompts,
4022            cache: cache.as_deref(),
4023            round: None,
4024        };
4025        let results = ask_json_wave::<Ranking>(
4026            judge_jobs,
4027            Arc::clone(&self.sem),
4028            retries,
4029            &self.roles.judge_roster,
4030            &ctx,
4031            &mut judge_losses,
4032            &mut self.state,
4033            &move |r: &Ranking| r.validate(&labels_for_check),
4034        )
4035        .await;
4036        if self.state.resource_block.is_some() {
4037            self.stop_for_resource()?;
4038            return Ok(false);
4039        }
4040
4041        // Refresh the judgement of every seat that ranked again.
4042        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4043        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4044            let agent_id = seat.agent.clone();
4045            self.state.seats.insert(seat.key.clone(), seat);
4046            let record = &mut self.state.judgements[j];
4047            match res {
4048                Ok((ranking, out)) => {
4049                    record.agent = agent_id;
4050                    record.ranking = ranking.normalized();
4051                    record.reasons = ranking.reasons;
4052                    record.strengths = ranking.strengths;
4053                    record.confidence = ranking.confidence;
4054                    record.failed = None;
4055                    record.duration_ms = out.duration_ms;
4056                    recovered.insert(j);
4057                    self.state.event(
4058                        "recover",
4059                        format!("judge {} ranked again after the limit", j + 1),
4060                    );
4061                }
4062                Err(e) => {
4063                    self.state
4064                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4065                }
4066            }
4067        }
4068
4069        // Re-ask the votes of the seats that recovered a ranking.
4070        let mut vote_jobs = Vec::new();
4071        let mut vote_pos: Vec<usize> = Vec::new();
4072        for &j in &recovered {
4073            let seat_key = format!("judge-{}", j + 1);
4074            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4075            let seat = self.seat(&seat_key, &spec.id);
4076            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4077            let text = if has_context(&spec, &seat, sessions) {
4078                prompt::final_vote(&labels, &language)
4079            } else {
4080                full.clone()
4081            };
4082            vote_jobs.push(SeatJob {
4083                spec,
4084                seat,
4085                prompt: text,
4086                cwd: root.join(seat_key),
4087                timeout,
4088                allow_write: false,
4089                sessions,
4090                artifacts: artifacts.clone(),
4091                stem: format!("vote-judge-{}-recover", j + 1),
4092                handover: Some(full),
4093            });
4094            vote_pos.push(j);
4095        }
4096        let allowed = labels.clone();
4097        let mut vote_losses = Vec::new();
4098        let vote_retries = self.state.config.graph.retries;
4099        let vote_cache = self.state.config.cache_dir();
4100        let ctx = WaveCtx {
4101            carry_seats: false,
4102            pauses: Some((&self.pause, &self.interrupt)),
4103            run: &run_id,
4104            node: "vote",
4105            prompts: &prompts,
4106            cache: vote_cache.as_deref(),
4107            round: None,
4108        };
4109        let votes = ask_json_wave::<FinalVote>(
4110            vote_jobs,
4111            Arc::clone(&self.sem),
4112            vote_retries,
4113            &self.roles.judge_roster,
4114            &ctx,
4115            &mut vote_losses,
4116            &mut self.state,
4117            &move |v: &FinalVote| match v.label() {
4118                Some(c) if allowed.contains(&c) => Ok(()),
4119                other => bail!("vote {other:?} is not one of {allowed:?}"),
4120            },
4121        )
4122        .await;
4123        if self.state.resource_block.is_some() {
4124            self.stop_for_resource()?;
4125            return Ok(false);
4126        }
4127        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4128            let agent_id = seat.agent.clone();
4129            self.state.seats.insert(seat.key.clone(), seat);
4130            match res {
4131                Ok((v, _)) => {
4132                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4133                        rec.vote = v.label();
4134                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4135                    } else {
4136                        self.state.votes.push(VoteRecord {
4137                            judge: j + 1,
4138                            agent: agent_id,
4139                            vote: v.label(),
4140                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4141                            changed: false,
4142                        });
4143                    }
4144                    self.state.event(
4145                        "recover",
4146                        format!("judge {} voted again after the limit", j + 1),
4147                    );
4148                }
4149                Err(e) => {
4150                    self.state
4151                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4152                }
4153            }
4154        }
4155
4156        // A seat that ranked again is present even if its re-vote failed —
4157        // `tally` falls back to the initial ranking's first choice — so clear
4158        // its quota loss. Seats that still fail keep theirs and stay absent.
4159        let recovered_keys: BTreeSet<String> = recovered
4160            .iter()
4161            .map(|&j| format!("judge-{}", j + 1))
4162            .collect();
4163        self.state
4164            .quota
4165            .retain(|q| !recovered_keys.contains(&q.seat));
4166        // A seat that hit the limit again is a fresh loss, not the old one:
4167        // replace the stale entry so the history stays one-per-seat and the
4168        // daemon can tell this attempt's loss from a previous session's.
4169        for loss in judge_losses.into_iter().chain(vote_losses) {
4170            if recovered_keys.contains(&loss.seat) {
4171                continue;
4172            }
4173            self.state.quota.retain(|q| q.seat != loss.seat);
4174            self.state.quota.push(loss);
4175        }
4176
4177        // Recompute the verdict from the refreshed panel.
4178        self.state.tally = None;
4179        self.tally()?;
4180        Ok(self
4181            .state
4182            .tally
4183            .as_ref()
4184            .map(|t| t.met_quorum)
4185            .unwrap_or(false))
4186    }
4187
4188    // ---------------------------------------------------------------- graft
4189
4190    /// Port the strengths the judges attributed to the losing candidates onto
4191    /// the winner, once, between the tally and the review loop.
4192    ///
4193    /// Best-effort by construction: it returns nothing and fails nothing. A
4194    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4195    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4196    /// the run carries on as if the phase were off. See [`graft`] for the
4197    /// decisions and `AGENTS.md` for the invariants.
4198    ///
4199    /// The one exception is a rollback that could not be confirmed: the
4200    /// rejected commit may still be the winner's tip, so the run is blocked
4201    /// (resumable; the record stays `Running`, which makes a resume retry the
4202    /// restore) instead of reviewing, gating and merging it.
4203    async fn graft(&mut self) -> Result<()> {
4204        let Err(e) = self.graft_inner().await else {
4205            return Ok(());
4206        };
4207        if e.is::<crate::repo_lock::ResourceBlocked>() {
4208            return Err(e);
4209        }
4210        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4211            self.state.status = RunStatus::Blocked;
4212            self.state.event("graft", format!("blocked: {e:#}"));
4213            self.state.save()?;
4214            return Err(e);
4215        }
4216        // An error after the `Running` record was saved may have left the
4217        // fixer's work in the tree: the record must not stay unsettled
4218        // (a later resume would reset past the review fixes), so confirm the
4219        // restore now, or block.
4220        if let Some(rec) = self.state.graft.clone()
4221            && !rec.outcome.settled()
4222        {
4223            if let Some(winner) = self.state.winner().cloned() {
4224                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4225                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4226                        return Err(re);
4227                    }
4228                    self.state.status = RunStatus::Blocked;
4229                    self.state.event("graft", format!("blocked: {re:#}"));
4230                    self.state.save()?;
4231                    return Err(re);
4232                }
4233            }
4234            if let Some(g) = self.state.graft.as_mut() {
4235                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4236            }
4237        }
4238        self.state.event("graft", format!("skipped: {e:#}"));
4239        if let Err(e) = self.state.save() {
4240            tracing::warn!("save after graft: {e:#}");
4241        }
4242        Ok(())
4243    }
4244
4245    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4246    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4247    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4248        let _coordination = self.coordinate(wt).await?;
4249        crate::recovery::restore_repair(wt, pre)
4250            .await
4251            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4252    }
4253
4254    /// A skip is a run event, recorded once however often the node is re-entered.
4255    fn graft_skipped(&mut self, why: &str) {
4256        let message = format!("skipped: {why}");
4257        if !self
4258            .state
4259            .events
4260            .iter()
4261            .any(|e| e.node == "graft" && e.message == message)
4262        {
4263            self.state.event("graft", message);
4264        }
4265    }
4266
4267    async fn graft_inner(&mut self) -> Result<()> {
4268        // A round that was started and never finished (crash, kill): the
4269        // fixer may have left anything in the tree, so put the winner back.
4270        if let Some(rec) = self.state.graft.clone()
4271            && !rec.outcome.settled()
4272        {
4273            // Reviews exist only if the graft ended without its record being
4274            // settled (a swallowed error): the tree has moved on since, and
4275            // a reset to `pre` would discard the review fixes.
4276            if self.state.reviews.is_empty()
4277                && let Some(winner) = self.state.winner().cloned()
4278            {
4279                self.graft_restore(&winner.worktree, &rec.pre).await?;
4280            }
4281            if rec.outcome == GraftOutcome::WaitingForRestore {
4282                self.state.graft = None;
4283                self.state.save()?;
4284            } else {
4285                if let Some(g) = self.state.graft.as_mut() {
4286                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4287                }
4288                self.state.event(
4289                    "graft",
4290                    "rolled back: the previous round was interrupted; the winner is untouched",
4291                );
4292                self.state.save()?;
4293                return Ok(());
4294            }
4295        }
4296        let plan = match graft::plan(&self.state) {
4297            Ok(plan) => plan,
4298            Err(skip) => {
4299                if skip != graft::Skip::AlreadyDone {
4300                    self.graft_skipped(skip.reason());
4301                    self.state.save()?;
4302                }
4303                return Ok(());
4304            }
4305        };
4306        let Some(winner) = self.state.winner().cloned() else {
4307            return Ok(());
4308        };
4309        let wt = winner.worktree.clone();
4310        if !git::is_clean(&wt).await? {
4311            self.graft_skipped("the winner's worktree has uncommitted changes");
4312            self.state.save()?;
4313            return Ok(());
4314        }
4315        let pre = git::rev_parse(&wt, "HEAD").await?;
4316        // Saved before the fixer is called: a stop from here on is repaired
4317        // by the branch above on the next entry.
4318        self.state.graft = Some(GraftRecord {
4319            pre: pre.clone(),
4320            post: None,
4321            lines: None,
4322            limit: None,
4323            agent: None,
4324            outcome: GraftOutcome::Running,
4325            requested: graft::requested(&plan),
4326            adoption: None,
4327            adoption_error: None,
4328        });
4329        self.state.event(
4330            "graft",
4331            format!(
4332                "porting strengths of candidate(s) {} onto {}",
4333                plan.strengths
4334                    .iter()
4335                    .map(|(l, _, _)| l.to_string())
4336                    .collect::<Vec<_>>()
4337                    .join(", "),
4338                plan.winner
4339            ),
4340        );
4341        self.state.save()?;
4342
4343        let fix_prompt = prompt::graft(
4344            &self.state.instruction,
4345            plan.winner,
4346            &plan.strengths,
4347            &self.state.config.graph.language,
4348        );
4349        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4350        let sessions = self.state.config.graph.sessions;
4351        let artifacts = agent::artifacts_dir(&self.state.dir());
4352        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4353        let (_, seat, out) = self
4354            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4355                SeatJob {
4356                    prompt: fix_prompt.clone(),
4357                    spec,
4358                    seat,
4359                    cwd: wt.clone(),
4360                    timeout,
4361                    allow_write: true,
4362                    sessions,
4363                    artifacts: artifacts.clone(),
4364                    stem: "graft".to_owned(),
4365                    handover: None,
4366                }
4367            })
4368            .await;
4369        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4370            // Preserve the need to restore earlier fallback edits without
4371            // confusing an unstarted seat with an interrupted invocation.
4372            self.state.graft.as_mut().expect("graft record").outcome =
4373                GraftOutcome::WaitingForRestore;
4374            return self.stop_for_resource();
4375        }
4376        let mut why: Option<String> = None;
4377        match &out {
4378            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4379                why = Some("the fixer produced nothing usable".to_owned());
4380            }
4381            AgentOutcome::Ok(_) => {}
4382            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4383            AgentOutcome::Quota(o) => {
4384                self.state.quota.push(QuotaLoss {
4385                    seat: seat.key.clone(),
4386                    node: "graft".to_owned(),
4387                    at: Timestamp::now(),
4388                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4389                });
4390                why = Some("rate limited (quota)".to_owned());
4391            }
4392            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4393                why = Some(format!(
4394                    "the fixer failed: {}",
4395                    e.lines()
4396                        .next()
4397                        .unwrap_or("")
4398                        .chars()
4399                        .take(160)
4400                        .collect::<String>()
4401                ));
4402            }
4403        }
4404        let agent_id = seat.agent.clone();
4405        self.state.seats.insert(seat.key.clone(), seat);
4406
4407        let mut post = None;
4408        let mut lines = None;
4409        let _coordination = self.coordinate(&wt).await?;
4410        if why.is_none() {
4411            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4412            if let Ok(r) = &rescued {
4413                self.state.note_withheld("graft", &r.withheld);
4414            }
4415            let head = git::rev_parse(&wt, "HEAD").await?;
4416            // What is measured must be what gets reviewed and gated: a failed
4417            // rescue, or anything still uncommitted (withheld lockfiles
4418            // aside, which are never part of the tree), rejects the graft.
4419            let dirty = match &rescued {
4420                Ok(r) => git::status_porcelain(&wt)
4421                    .await
4422                    .map(|st| {
4423                        st.lines().any(|l| {
4424                            !r.withheld
4425                                .iter()
4426                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4427                        })
4428                    })
4429                    .unwrap_or(true),
4430                Err(_) => true,
4431            };
4432            if dirty {
4433                why = Some("the fixer left changes that could not be committed".to_owned());
4434            } else if head == pre {
4435                why = Some("the fixer changed nothing".to_owned());
4436            } else {
4437                lines = git::changed_lines(&wt, &pre, &head).await;
4438                post = Some(head);
4439            }
4440        }
4441        let (adoption, adoption_error) = match &out {
4442            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4443            _ => (None, None),
4444        };
4445        let requested = graft::requested(&plan);
4446        let outcome = match why {
4447            None => {
4448                self.state.event(
4449                    "graft",
4450                    format!(
4451                        "applied: {}",
4452                        lines.map_or_else(
4453                            || "changed lines unavailable".to_owned(),
4454                            |n| format!("{n} changed lines")
4455                        )
4456                    ),
4457                );
4458                GraftOutcome::Applied
4459            }
4460            Some(why) => {
4461                // Not settled until the restore is confirmed: the record
4462                // stays `Running`, so a resume retries it.
4463                crate::recovery::restore_repair(&wt, &pre)
4464                    .await
4465                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4466                self.state.event(
4467                    "graft",
4468                    format!("rolled back: {why}; the winner is untouched"),
4469                );
4470                GraftOutcome::RolledBack(why)
4471            }
4472        };
4473        self.state.graft = Some(GraftRecord {
4474            pre,
4475            post,
4476            lines,
4477            limit: None,
4478            agent: Some(agent_id),
4479            outcome,
4480            requested,
4481            adoption,
4482            adoption_error,
4483        });
4484        self.state.save()?;
4485        Ok(())
4486    }
4487
4488    // ----------------------------------------------------------------- fold
4489
4490    async fn fold_losers(&mut self) -> Result<()> {
4491        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4492            return Ok(());
4493        };
4494        if !self.state.candidate_preservation.is_empty() {
4495            bail!("candidate preservation incomplete; refusing cleanup");
4496        }
4497        let repo = self.state.repo.clone();
4498        let _coordination = self.coordinate(&repo).await?;
4499        let mut folded = Vec::new();
4500        for i in 0..self.state.candidates.len() {
4501            let c = &self.state.candidates[i];
4502            if c.label == winner || c.folded {
4503                continue;
4504            }
4505            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4506            remove_preserved_worktree(&repo, &wt).await?;
4507            if git::branch_exists(&repo, &branch).await?
4508                && !git::branch_delete(&repo, &branch).await?
4509            {
4510                bail!("candidate branch removal refused");
4511            }
4512            self.state.candidates[i].folded = true;
4513            self.state.save()?;
4514            folded.push(label.to_string());
4515        }
4516        // The judges are finished; their checkouts are pure cost from here.
4517        let root = self.state.worktree_root();
4518        for j in 1..=self.roles.judges.len() {
4519            let wt = root.join(format!("judge-{j}"));
4520            if wt.exists() {
4521                remove_preserved_worktree(&repo, &wt).await?;
4522            }
4523        }
4524        // The design-deliberation stage is finished by the time a tally
4525        // exists — same reasoning as the judges above.
4526        if self.state.config.graph.advise {
4527            for k in 1..=self.state.config.graph.advisors {
4528                let wt = root.join(format!("advisor-{k}"));
4529                if wt.exists() {
4530                    remove_preserved_worktree(&repo, &wt).await?;
4531                }
4532            }
4533        }
4534        if !folded.is_empty() {
4535            self.state
4536                .event("fold", format!("folded candidates {}", folded.join(", ")));
4537            self.state.save()?;
4538        }
4539        Ok(())
4540    }
4541
4542    // ------------------------------------------------------------ base sync
4543
4544    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4545    /// anything verifies it.
4546    ///
4547    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4548    /// read whatever is checked out in the winner's worktree. Left alone that
4549    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4550    /// it when the run *branched* - and a run takes long enough that the base
4551    /// has usually moved by the time it gets here. A gate that ran there
4552    /// answers "green on the commit this run started from", not "green on
4553    /// what is about to land", and the difference showed up three times in
4554    /// one day as a green run whose merge would have reverted a file another
4555    /// pull request had already landed.
4556    ///
4557    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4558    /// `land::Step::Rebase` calls, rather than a second implementation of the
4559    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4560    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4561    /// doc warns about ("two answers to a question nobody notices until a
4562    /// diff is wrong").
4563    ///
4564    /// A conflict is not the end of the road: the standing rebase is handed
4565    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4566    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4567    /// land). Once it finishes, review and the gate run as usual on the
4568    /// rebased tree, which is where a breakage the new base caused is caught
4569    /// by the ordinary gate-fix round. magi resolves nothing itself.
4570    ///
4571    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4572    /// `state.base_sync.attempts`, is how many times the base is *rebased
4573    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4574    /// *conflict* was given to a fixer. When the fixer cannot finish the
4575    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4576    /// what was tried (rounds spent, paths still conflicted) and the branch
4577    /// and worktree stay exactly as they were - untouched, for a person to
4578    /// look at - which is also what makes re-entering this function
4579    /// afterwards a no-op instead of a second attempt at the same wall. A
4580    /// push failure ends the same way.
4581    async fn sync_to_base(&mut self) -> Result<()> {
4582        if self.state.status == RunStatus::AlreadyInBase {
4583            return Ok(());
4584        }
4585        let conflicted = self
4586            .state
4587            .base_sync
4588            .as_ref()
4589            .is_some_and(|s| s.conflict.is_some());
4590        let Some(winner) = self.state.winner().cloned() else {
4591            return Ok(());
4592        };
4593
4594        let repo = self.state.repo.clone();
4595        let remote = self.state.config.merge.remote.clone();
4596        let base_branch = self.state.base_branch.clone();
4597        let tracking = format!("{remote}/{base_branch}");
4598
4599        git::fetch(&repo, &remote, &base_branch).await.ok();
4600        // No network, or the remote never had this branch: the run already
4601        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4602        // otherwise), and one that got this far is not blocked by a fetch
4603        // that fails now. It just has no newer tip to compare against.
4604        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4605            return Ok(());
4606        };
4607
4608        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4609        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4610        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4611
4612        // Before any rebase, and before a recorded conflict is honoured: a
4613        // branch whose change reached the base under other commit ids has
4614        // nothing to rebase and nothing to conflict with, and a run that
4615        // already stopped on that phantom conflict recovers here on resume.
4616        // `behind == 0` with head == tip is a branch the base has since taken
4617        // in whole, whether or not a conflict was ever recorded: the ancestry
4618        // proof must still run (`classify` ignores a head still on the start
4619        // commit).
4620        if (behind > 0 || conflicted || head == tip)
4621            && self
4622                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4623                .await?
4624        {
4625            return Ok(());
4626        }
4627        if conflicted {
4628            return Ok(());
4629        }
4630
4631        if behind == 0 {
4632            // A fixer-finished rebase moves the branch ref before the
4633            // winner's worktree is told (`sync_to_head` below). A run that
4634            // died in between resumes here with `behind == 0` and a tree still
4635            // holding the pre-rebase files, which review and the gate would
4636            // then read. That state is exactly: HEAD moved off the tip the
4637            // rebase started from, yet the tree is still identical to that
4638            // tip. A tree with edits of its own differs from it, so nothing
4639            // is thrown away.
4640            if let Some(from) = self
4641                .state
4642                .rebase_fixes
4643                .iter()
4644                .rev()
4645                .find_map(|r| r.from.clone())
4646                && from != head
4647                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4648                    .await
4649                    .is_ok_and(|o| o.ok())
4650            {
4651                git::sync_to_head(&winner.worktree).await?;
4652            }
4653            // An earlier attempt may have rebased the branch locally and died
4654            // before pushing it (only the fresh-rebase arm below pushes).
4655            // Publish it now, so the plain push at PR time is not refused as
4656            // a non-fast-forward. A run already holding a recorded conflict
4657            // never reaches here; that case is out of scope.
4658            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4659            if let Some(why) = &conflict {
4660                self.state.status = RunStatus::Blocked;
4661                self.state.event("land", why.clone());
4662            }
4663            self.state.base_sync = Some(BaseSync {
4664                tip,
4665                behind: 0,
4666                attempts,
4667                conflict,
4668                already_in: None,
4669            });
4670            self.state.save()?;
4671            return Ok(());
4672        }
4673
4674        if attempts >= BASE_SYNC_ROUNDS {
4675            let why = format!(
4676                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4677                 rebase(s); rebasing again would only race it",
4678                winner.branch
4679            );
4680            self.state.status = RunStatus::Blocked;
4681            self.state.base_sync = Some(BaseSync {
4682                tip,
4683                behind,
4684                attempts,
4685                conflict: Some(why.clone()),
4686                already_in: None,
4687            });
4688            self.state.event("land", why);
4689            self.state.save()?;
4690            return Ok(());
4691        }
4692
4693        self.state.event(
4694            "land",
4695            format!(
4696                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4697                winner.branch
4698            ),
4699        );
4700        self.state.save()?;
4701
4702        // The remote's copy of the branch, read now and only if the fetch
4703        // really succeeded (a stale tracking ref must never pin a lease). It is
4704        // pushed over after a rebase only when it is a commit this branch
4705        // already contains, by ancestry or by patch (an earlier rebase of ours
4706        // that never reached the remote): anything else is somebody else's work.
4707        let branch_tracking = format!("{remote}/{}", winner.branch);
4708        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4709        let remote_tip = if self.state.task_pr_branch().is_none()
4710            && matches!(&fetched_branch, Ok(o) if o.ok())
4711        {
4712            git::rev_parse(&repo, &branch_tracking).await.ok()
4713        } else {
4714            None
4715        };
4716        // A remote tip this branch does not contain is somebody else's work:
4717        // rebasing would leave a local tip that can never be pushed. Stop
4718        // before touching anything and say so.
4719        if let Some(theirs) = &remote_tip
4720            && !git::is_ancestor(&repo, theirs, &head).await
4721            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4722                .await
4723                .is_ok_and(|missing| missing.is_empty())
4724        {
4725            let why = format!(
4726                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4727                 them",
4728                short(theirs),
4729                winner.branch
4730            );
4731            self.state.status = RunStatus::Blocked;
4732            self.state.base_sync = Some(BaseSync {
4733                tip,
4734                behind,
4735                attempts,
4736                conflict: Some(why.clone()),
4737                already_in: None,
4738            });
4739            self.state.event("land", why);
4740            self.state.save()?;
4741            return Ok(());
4742        }
4743
4744        let scratch = self.state.dir().join("base-sync");
4745        let rebased = match crate::rebase::rebase_with_fixer(
4746            &mut self.state,
4747            &scratch,
4748            &winner.branch,
4749            &tracking,
4750        )
4751        .await
4752        {
4753            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4754            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4755            Err(e) => Err(e),
4756        };
4757        let attempts = attempts + 1;
4758        match rebased {
4759            Ok(None) => {
4760                // The branch ref moved, but a worktree that already had it
4761                // checked out (the winner's) was not told; sync its index and
4762                // files before anything reads them.
4763                git::sync_to_head(&winner.worktree).await?;
4764                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4765                let mut conflict = None;
4766                if let Some(pinned) = &remote_tip {
4767                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4768                    match pushed {
4769                        Ok(o) if o.ok() => self.state.event(
4770                            "land",
4771                            format!("pushed rebased {} to {remote}", winner.branch),
4772                        ),
4773                        Ok(o) => {
4774                            conflict = Some(format!(
4775                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4776                                winner.branch,
4777                                short(pinned),
4778                                o.stderr.chars().take(600).collect::<String>()
4779                            ));
4780                        }
4781                        Err(e) => {
4782                            conflict = Some(format!(
4783                                "rebased {} locally but could not push it: {e:#}",
4784                                winner.branch
4785                            ));
4786                        }
4787                    }
4788                }
4789                if let Some(why) = &conflict {
4790                    self.state.status = RunStatus::Blocked;
4791                    self.state.event("land", why.clone());
4792                }
4793                self.state.base_sync = Some(BaseSync {
4794                    tip: tip.clone(),
4795                    behind: 0,
4796                    attempts,
4797                    conflict,
4798                    already_in: None,
4799                });
4800                self.state
4801                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4802            }
4803            Ok(Some(conflict)) => {
4804                let why = format!(
4805                    "{} conflicts with {tracking} and did not rebase: {}",
4806                    winner.branch,
4807                    conflict.chars().take(600).collect::<String>()
4808                );
4809                self.state.status = RunStatus::Blocked;
4810                self.state.base_sync = Some(BaseSync {
4811                    tip,
4812                    behind,
4813                    attempts,
4814                    conflict: Some(why.clone()),
4815                    already_in: None,
4816                });
4817                self.state.event("land", why);
4818            }
4819            // A refused Git write or a busy repository: nothing is wrong with
4820            // the branch. The scratch rebase stays standing and the run
4821            // resumes into it (`base_sync` must not claim a conflict).
4822            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4823                self.state.save()?;
4824                return Ok(());
4825            }
4826            Err(e) => {
4827                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4828                self.state.status = RunStatus::Blocked;
4829                self.state.base_sync = Some(BaseSync {
4830                    tip,
4831                    behind,
4832                    attempts,
4833                    conflict: Some(why.clone()),
4834                    already_in: None,
4835                });
4836                self.state.event("land", why);
4837            }
4838        }
4839        self.state.save()?;
4840        Ok(())
4841    }
4842
4843    /// Push a branch an earlier attempt rebased locally but never published,
4844    /// pinned to the remote tip read right after a successful fetch. Returns
4845    /// the reason when the run must stop; `None` when there was nothing to do
4846    /// (no remote copy, the same tip, or a remote copy this branch already
4847    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4848    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4849        // Task competitions publish only after the PR text gate, to their
4850        // public head; never resurrect a candidate branch on the remote.
4851        if self.state.task_pr_branch().is_some() {
4852            return None;
4853        }
4854        let repo = self.state.repo.clone();
4855        let remote = self.state.config.merge.remote.clone();
4856        let fetched = git::fetch(&repo, &remote, branch).await;
4857        if !matches!(&fetched, Ok(o) if o.ok()) {
4858            return None;
4859        }
4860        let branch_tracking = format!("{remote}/{branch}");
4861        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4862        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4863            return None;
4864        }
4865        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4866            .await
4867            .is_ok_and(|missing| missing.is_empty())
4868        {
4869            return Some(format!(
4870                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4871                 them",
4872                short(&theirs)
4873            ));
4874        }
4875        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4876            Ok(o) if o.ok() => {
4877                self.state
4878                    .event("land", format!("pushed rebased {branch} to {remote}"));
4879                None
4880            }
4881            Ok(o) => Some(format!(
4882                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4883                 someone may have pushed): {}",
4884                short(&theirs),
4885                o.stderr.chars().take(600).collect::<String>()
4886            )),
4887            Err(e) => Some(format!(
4888                "{branch} is rebased locally but could not be pushed: {e:#}"
4889            )),
4890        }
4891    }
4892
4893    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4894    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4895    /// whether it did.
4896    ///
4897    /// Checked only when the base is ahead of the branch. A failing check is
4898    /// "not proven" - the ordinary rebase path then decides - never a reason to
4899    /// stop the run.
4900    ///
4901    /// The remote copy of the branch is held to the same standard as the local
4902    /// one: if it carries a tip this worktree does not, that tip must itself be
4903    /// proven in the base, or nothing is settled (a pull request would
4904    /// otherwise be closed over commits nobody checked). The pull request is
4905    /// closed *before* the terminal status is saved; if that fails for a
4906    /// reason other than a refusal (no network, a `gh` error) the run is left
4907    /// `Blocked` with the reason as its conflict, which a resume retries -
4908    /// the same recovery a phantom conflict gets.
4909    async fn settle_already_in(
4910        &mut self,
4911        branch: &str,
4912        tip: &str,
4913        head: &str,
4914        attempts: usize,
4915        behind: usize,
4916    ) -> Result<bool> {
4917        let repo = self.state.repo.clone();
4918        let remote = self.state.config.merge.remote.clone();
4919        let start = self.state.base_commit.clone();
4920        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4921            Ok(Some(e)) => e,
4922            Ok(None) => return Ok(false),
4923            Err(e) => {
4924                tracing::warn!("already-in-base check for {branch}: {e:#}");
4925                return Ok(false);
4926            }
4927        };
4928        let mut verified = vec![head.to_owned()];
4929        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4930        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4931        if matches!(&fetched, Ok(o) if o.ok())
4932            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4933            && theirs != head
4934        {
4935            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4936                Ok(Some(_)) => verified.push(theirs),
4937                _ => return Ok(false),
4938            }
4939        }
4940        let base_branch = self.state.base_branch.clone();
4941        let message = format!(
4942            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4943             land",
4944            evidence.names(),
4945            evidence.proof.as_str()
4946        );
4947        let closed =
4948            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4949                .await;
4950        match closed {
4951            Ok(Ok(url)) => self
4952                .state
4953                .event("land", format!("closed {url}: superseded on {base_branch}")),
4954            Ok(Err(why)) => self
4955                .state
4956                .event("land", format!("did not close a pull request: {why}")),
4957            Err(e) => {
4958                let why = format!(
4959                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4960                     not be closed ({e:#}); resume to retry"
4961                );
4962                self.state.status = RunStatus::Blocked;
4963                self.state.base_sync = Some(BaseSync {
4964                    tip: tip.to_owned(),
4965                    behind,
4966                    attempts,
4967                    conflict: Some(why.clone()),
4968                    already_in: None,
4969                });
4970                self.state.event("land", why);
4971                self.state.save()?;
4972                return Ok(true);
4973            }
4974        }
4975        self.state.status = RunStatus::AlreadyInBase;
4976        self.state.base_sync = Some(BaseSync {
4977            tip: tip.to_owned(),
4978            behind,
4979            attempts,
4980            conflict: None,
4981            already_in: Some(evidence),
4982        });
4983        self.state.event("land", message);
4984        self.state.save()?;
4985        self.settle_questions();
4986        Ok(true)
4987    }
4988
4989    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
4990    /// last landed the winner on, once it has run, else the commit the run
4991    /// branched from.
4992    ///
4993    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
4994    /// and `vote` all happen before there is a winner to rebase, so they
4995    /// compare every candidate against the branch point on purpose, and a
4996    /// base that moves after they are already done cannot change an answer
4997    /// they already gave.
4998    fn landing_base(&self) -> String {
4999        self.state
5000            .base_sync
5001            .as_ref()
5002            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5003    }
5004
5005    // ------------------------------------------------------- operator fix
5006
5007    /// Route specific, already-recorded review findings to a fixer for a
5008    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5009    /// entry point.
5010    ///
5011    /// Distinct from `review_loop`'s own fix step in three ways: it never
5012    /// runs a reviewer wave, it never spends review-round budget, and what
5013    /// happened is recorded as an [`OperatorFixRequest`] appended to
5014    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5015    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5016    /// and vote must never be rewritten to look like a manufactured blocking
5017    /// verdict.
5018    ///
5019    /// Only meaningful once review has actually concluded: `Ready` (handed
5020    /// off with findings still open, or simply concluded clean while minor
5021    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5022    /// gate failed). Everything else is refused: a run still in progress
5023    /// should simply be resumed, and a `Merged` run's branch has already
5024    /// landed — reopening *this* run's own record cannot change that, so the
5025    /// answer there is a fresh `magi review <branch>`.
5026    ///
5027    /// A real commit here re-verifies through a fresh, ordinary review-only
5028    /// run on the same branch ([`Self::review`]) rather than reopening this
5029    /// run's own `review_loop`: once any round in this run's history went
5030    /// clean, `review_conclusion` treats that as permanent by design (the
5031    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5032    /// way to force one more genuine reviewer wave out of *this* run without
5033    /// either rewriting history or weakening that guarantee for every other
5034    /// caller. A review-only run costs nothing extra — no implementation, no
5035    /// judging, no vote — and exercises the exact same review → verify →
5036    /// gate → (human) merge path, unmodified.
5037    pub async fn fix_selected(
5038        &mut self,
5039        ids: &[String],
5040        reason: &str,
5041        allow_stale: bool,
5042    ) -> Result<()> {
5043        let reason = reason.trim();
5044        if reason.is_empty() {
5045            bail!("a fix request needs a reason — that is the operator's own record of why");
5046        }
5047        if ids.is_empty() {
5048            bail!("no finding id given");
5049        }
5050        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5051            bail!(
5052                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5053                 has already concluded — can be given a targeted fix. A run still \
5054                 in progress should simply be resumed; a `merged` run's branch has \
5055                 already landed, so its answer is a fresh `magi review <branch>`, \
5056                 not reopening this run's own record",
5057                self.state.id,
5058                self.state.status.as_str()
5059            );
5060        }
5061        let Some(winner) = self.state.winner().cloned() else {
5062            bail!("run {} has no winning candidate to fix", self.state.id);
5063        };
5064        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5065            bail!(
5066                "branch `{}` no longer exists; this run cannot be extended",
5067                winner.branch
5068            );
5069        }
5070        let home = crate::run::home();
5071        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5072            bail!(
5073                "run {} is currently being worked on by another magi process",
5074                self.state.id
5075            );
5076        }
5077        // Held for the rest of this call, including the follow-up review
5078        // below: two `magi fix` invocations against the same run must not
5079        // both reach the worktree manipulation further down, which would
5080        // otherwise race to remove and recreate the same directory — see
5081        // [`FixClaim`]'s own doc.
5082        let _claim = FixClaim::acquire(&self.state.dir())?;
5083
5084        // Resolve every id before spending anything — an unknown id refuses
5085        // the whole request rather than silently dropping it — and dedup
5086        // while keeping the operator's own order.
5087        let mut seen = BTreeSet::new();
5088        let mut findings = Vec::new();
5089        let mut missing = Vec::new();
5090        for id in ids {
5091            if !seen.insert(id.clone()) {
5092                continue;
5093            }
5094            match self.state.finding(id) {
5095                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5096                    id: f.id.clone(),
5097                    severity: f.severity,
5098                    reviewer_vote: rec.vote,
5099                    round: round.round,
5100                    round_head: round.head.clone(),
5101                    reviewer: rec.reviewer,
5102                    agent: rec.agent.clone(),
5103                    file: f.file.clone(),
5104                    line: f.line,
5105                    title: f.title.clone(),
5106                    detail: f.detail.clone(),
5107                    outcome: OperatorFixOutcome::Pending,
5108                }),
5109                None => missing.push(id.clone()),
5110            }
5111        }
5112        if !missing.is_empty() {
5113            bail!(
5114                "unknown finding id(s): {}; nothing was changed",
5115                missing.join(", ")
5116            );
5117        }
5118
5119        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5120        let stale_details: Vec<(String, String)> = findings
5121            .iter()
5122            .filter(|f| f.round_head != head_at_request)
5123            .map(|f| (f.id.clone(), f.round_head.clone()))
5124            .collect();
5125        let stale = !stale_details.is_empty();
5126        if stale && !allow_stale {
5127            bail!(
5128                "the branch has moved since some finding(s) were raised — {} — now \
5129                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5130                stale_details
5131                    .iter()
5132                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5133                    .collect::<Vec<_>>()
5134                    .join(", "),
5135                short(&head_at_request)
5136            );
5137        }
5138
5139        let request = OperatorFixRequest {
5140            requested_at: Timestamp::now(),
5141            reason: reason.to_owned(),
5142            findings,
5143            head_at_request: head_at_request.clone(),
5144            allow_stale,
5145            stale,
5146            fix: None,
5147            result_head: None,
5148            follow_up_review_run: None,
5149        };
5150        self.state.event(
5151            "fix",
5152            format!(
5153                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5154                request.findings.len(),
5155                request
5156                    .findings
5157                    .iter()
5158                    .map(|f| f.id.as_str())
5159                    .collect::<Vec<_>>()
5160                    .join(", "),
5161            ),
5162        );
5163        // Recorded now, before any worktree work or the fixer call itself —
5164        // and re-saved at each checkpoint below: a crash at any point after
5165        // this (mid fixer call, mid follow-up review) must not lose the fact
5166        // that this was requested, for which findings, and why. Everything
5167        // past this point reads and writes through `request_index` rather
5168        // than a local variable, since `request` itself is moved here.
5169        self.state.operator_fixes.push(request);
5170        self.state.save()?;
5171        let request_index = self.state.operator_fixes.len() - 1;
5172
5173        // A fresh, dedicated worktree for this one call, never the winner's
5174        // own worktree in place: that one may already be gone (folded away),
5175        // and reusing it in place would leave the branch checked out there
5176        // when the follow-up review below tries to check it out again. Freed
5177        // immediately after, either way — but only once confirmed clean:
5178        // `worktree_remove` is a `git worktree remove --force`, which would
5179        // otherwise discard uncommitted work left there by the operator or
5180        // another process before this had a chance to even look at it.
5181        let repo = self.state.repo.clone();
5182        let coordination = self.coordinate(&repo).await?;
5183        if winner.worktree.exists() {
5184            // Lockfiles a rescue commit withheld stay untracked on purpose and
5185            // are already recorded; they are not the operator's work to protect.
5186            let dirty = git::git(
5187                &winner.worktree,
5188                &["status", "--porcelain", "--untracked-files=all"],
5189            )
5190            .await?;
5191            let only_withheld = dirty.lines().all(|l| {
5192                l.strip_prefix("?? ")
5193                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5194            });
5195            if !only_withheld {
5196                bail!(
5197                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5198                     discard them first",
5199                    winner.worktree.display()
5200                );
5201            }
5202            git::worktree_remove(&self.state.repo, &winner.worktree)
5203                .await
5204                .ok();
5205        }
5206        let fix_worktree = self.state.worktree_root().join("operator-fix");
5207        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5208        git::git(
5209            &self.state.repo,
5210            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5211        )
5212        .await
5213        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5214        if !git::is_clean(&fix_worktree).await? {
5215            git::worktree_remove(&self.state.repo, &fix_worktree)
5216                .await
5217                .ok();
5218            bail!(
5219                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5220                winner.branch
5221            );
5222        }
5223        let repo_for_setup = self.state.repo.clone();
5224        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5225            Ok(()) => {
5226                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5227            }
5228            Err(e) => Err(e),
5229        };
5230        if let Err(e) = setup {
5231            git::worktree_remove(&self.state.repo, &fix_worktree)
5232                .await
5233                .ok();
5234            return Err(e);
5235        }
5236
5237        let run_id = self.state.id.clone();
5238        let prompts = self.state.config.prompts.clone();
5239        let language = self.state.config.graph.language.clone();
5240        let sessions = self.state.config.graph.sessions;
5241        let artifacts = agent::artifacts_dir(&self.state.dir());
5242        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5243            .findings
5244            .iter()
5245            .map(|f| Finding {
5246                id: f.id.clone(),
5247                severity: f.severity,
5248                file: f.file.clone(),
5249                line: f.line,
5250                title: f.title.clone(),
5251                detail: f.detail.clone(),
5252            })
5253            .collect();
5254        let fix_prompt = prompt::operator_fix_in(
5255            &self.state.instruction,
5256            &finding_list,
5257            reason,
5258            &stale_details,
5259            &head_at_request,
5260            &language,
5261            &self.state.config.graph.github_language,
5262        );
5263        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5264        drop(coordination);
5265        let (job, seat, out) = self
5266            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5267                prompt: fix_prompt.clone(),
5268                spec,
5269                seat,
5270                cwd: fix_worktree.clone(),
5271                timeout,
5272                allow_write: true,
5273                sessions,
5274                artifacts: artifacts.clone(),
5275                stem: "operator-fix".to_owned(),
5276                handover: None,
5277            })
5278            .await;
5279        let agent_id = seat.agent.clone();
5280
5281        let mut fix = FixRecord {
5282            agent: agent_id,
5283            addressed: Vec::new(),
5284            rejected: Vec::new(),
5285            notes: String::new(),
5286            committed: false,
5287            failed: None,
5288            duration_ms: 0,
5289            continuation: None,
5290        };
5291        let mut final_seat = seat.clone();
5292        match out {
5293            AgentOutcome::Ok(o) => {
5294                fix.duration_ms = o.duration_ms;
5295                let parsed = verdict::extract_json::<FixReport>(&o.text);
5296                let incomplete_reason = match &parsed {
5297                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5298                        "the reply parsed, but it reported a command whose own CLI \
5299                         never confirmed an exit status"
5300                            .to_owned(),
5301                    ),
5302                    Ok(_) => None,
5303                    Err(e) => Some(e.to_string()),
5304                };
5305                match incomplete_reason {
5306                    None => {
5307                        let report = parsed.expect("checked Ok above");
5308                        fix.addressed = report.addressed;
5309                        fix.rejected = report.rejected;
5310                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5311                    }
5312                    Some(reason) => {
5313                        let (resumed_seat, resolved, failure, cont) = self
5314                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5315                            .await;
5316                        fix.duration_ms += cont.cumulative_wait_ms;
5317                        fix.continuation = Some(cont);
5318                        final_seat = resumed_seat;
5319                        match resolved {
5320                            Some(report) => {
5321                                fix.addressed = report.addressed;
5322                                fix.rejected = report.rejected;
5323                                fix.notes =
5324                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5325                            }
5326                            None => fix.failed = failure,
5327                        }
5328                    }
5329                }
5330            }
5331            AgentOutcome::Dropped(o) => {
5332                fix.duration_ms = o.duration_ms;
5333                let why = o
5334                    .dropped
5335                    .as_ref()
5336                    .map(|d| d.why.as_str())
5337                    .unwrap_or("the CLI ended the stream without delivering its answer");
5338                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5339            }
5340            AgentOutcome::Quota(o) => {
5341                self.state.quota.push(QuotaLoss {
5342                    seat: final_seat.key.clone(),
5343                    node: "fix".to_owned(),
5344                    at: Timestamp::now(),
5345                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5346                });
5347                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5348            }
5349            AgentOutcome::Failed(e) => fix.failed = Some(e),
5350            AgentOutcome::ResourceBlocked(e) => {
5351                fix.failed = Some(format!("the fixer was not started: {e}"));
5352            }
5353        }
5354        if fix.continuation.is_none() {
5355            fix.continuation = Some(ContinuationRecord::not_needed());
5356        }
5357        self.state.seats.insert(final_seat.key.clone(), final_seat);
5358
5359        let rescue_message = format!(
5360            "magi: operator-selected fix ({}) (uncommitted work)",
5361            self.state.operator_fixes[request_index]
5362                .findings
5363                .iter()
5364                .map(|f| f.id.as_str())
5365                .collect::<Vec<_>>()
5366                .join(", ")
5367        );
5368        let repo = self.state.repo.clone();
5369        let coordination = self.coordinate(&repo).await?;
5370        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5371            self.state.note_withheld("fix", &r.withheld);
5372        }
5373        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5374        fix.committed = after != head_at_request;
5375        git::worktree_remove(&self.state.repo, &fix_worktree)
5376            .await
5377            .ok();
5378        drop(coordination);
5379
5380        self.state.event(
5381            "fix",
5382            match &fix.failed {
5383                Some(reason) => format!(
5384                    "operator fix: adoption report was lost ({reason}); {}",
5385                    if fix.committed {
5386                        "committed"
5387                    } else {
5388                        "NO new commit"
5389                    }
5390                ),
5391                None => format!(
5392                    "operator fix: {} addressed, {} rejected, {}",
5393                    fix.addressed.len(),
5394                    fix.rejected.len(),
5395                    if fix.committed {
5396                        "committed"
5397                    } else {
5398                        "NO new commit"
5399                    }
5400                ),
5401            },
5402        );
5403
5404        // Every selected finding gets an outcome — never left `Pending` once
5405        // the fixer's own turn is over. A report that never came back at all
5406        // marks every one of them `Unreported`, not silently "not addressed":
5407        // quota, a dropped stream, or an exhausted continuation are gaps in
5408        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5409        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5410        for f in &mut self.state.operator_fixes[request_index].findings {
5411            f.outcome = if fix.failed.is_some() {
5412                OperatorFixOutcome::Unreported
5413            } else if fix.addressed.contains(&f.id) {
5414                OperatorFixOutcome::Addressed
5415            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5416                OperatorFixOutcome::Rejected { why: r.why.clone() }
5417            } else {
5418                OperatorFixOutcome::Unreported
5419            };
5420        }
5421
5422        let committed = fix.committed;
5423        if committed {
5424            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5425        }
5426        self.state.operator_fixes[request_index].fix = Some(fix);
5427        // Saved again now that the fixer's own outcome is final, on top of
5428        // the save right after the request was first pushed above.
5429        self.state.save()?;
5430
5431        if committed {
5432            self.state.event(
5433                "fix",
5434                format!(
5435                    "operator fix committed {}; opening a follow-up review-only run",
5436                    short(&after)
5437                ),
5438            );
5439            // The operator asked for the fix, and the follow-up serves whatever
5440            // task the run it follows served.
5441            let origin =
5442                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5443            match Self::review(
5444                &self.state.repo,
5445                &winner.branch,
5446                self.state.config.clone(),
5447                origin,
5448            )
5449            .await
5450            {
5451                Ok(mut follow_up) => {
5452                    follow_up.state.event(
5453                        "start",
5454                        format!(
5455                            "requested by an operator fix on run {} for finding(s) {}",
5456                            self.state.id,
5457                            self.state.operator_fixes[request_index]
5458                                .findings
5459                                .iter()
5460                                .map(|f| f.id.as_str())
5461                                .collect::<Vec<_>>()
5462                                .join(", "),
5463                        ),
5464                    );
5465                    follow_up.state.save()?;
5466                    let follow_up_id = follow_up.state.id.clone();
5467                    // The follow-up is a run like any other: it belongs to the
5468                    // task of the run it follows, or to one filed for it.
5469                    let adopted = match crate::direct::adopt(
5470                        &follow_up.state,
5471                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5472                    ) {
5473                        Ok(a) => a,
5474                        Err(e) => {
5475                            // An ownerless run must not spend agent calls; it
5476                            // stays saved, and `magi run --resume` adopts it.
5477                            self.state.event(
5478                                "fix",
5479                                format!(
5480                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5481                                ),
5482                            );
5483                            self.state.save()?;
5484                            return Ok(());
5485                        }
5486                    };
5487                    let executed = follow_up.execute().await;
5488                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5489                    if let Some(a) = adopted {
5490                        a.finish(&follow_up.state, executed);
5491                    }
5492                    if let Some(e) = failure {
5493                        self.state.event(
5494                            "fix",
5495                            format!(
5496                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5497                            ),
5498                        );
5499                    }
5500                    self.state.operator_fixes[request_index].follow_up_review_run =
5501                        Some(follow_up_id);
5502                }
5503                Err(e) => {
5504                    self.state.event(
5505                        "fix",
5506                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5507                    );
5508                }
5509            }
5510            self.state.save()?;
5511        }
5512
5513        Ok(())
5514    }
5515
5516    // --------------------------------------------------------------- review
5517
5518    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5519    /// names, in order, each at most once, moving on only when the call
5520    /// advances (an error, a quota hit or nothing usable - the same decision
5521    /// point `agent::chain_advances` is for the other chained roles).
5522    ///
5523    /// Each agent gets its own seat from `Runner::seat`: the same agent
5524    /// continues its conversation (the winner's own implementer seat when it
5525    /// is the winner's author, now that the competition is over), another
5526    /// takes a fresh one so the full prompt is sent again. A handover is
5527    /// recorded under `node`, which is what makes the fallback stick for the
5528    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5529    /// is returned, so an exhausted chain reads exactly like a single failed
5530    /// fixer: one quota loss, the same wording, the same refund. An earlier
5531    /// attempt's edits are left in the tree and judged, with the final
5532    /// attempt's, by what git says afterwards.
5533    ///
5534    /// Returns the job that produced the outcome, for `continue_fix_report`.
5535    async fn ask_fixer(
5536        &mut self,
5537        winner: &Candidate,
5538        node: &'static str,
5539        round: Option<usize>,
5540        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5541    ) -> (SeatJob, SeatState, AgentOutcome) {
5542        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5543        self.ask_fixer_among(attempts, node, round, None, build)
5544            .await
5545    }
5546
5547    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5548    /// worktree and a commit: before the next agent is asked after one that
5549    /// did not satisfy the call, the worktree is put back on that commit, so
5550    /// a failed agent's partial edits are not carried into the next one.
5551    async fn ask_fixer_among(
5552        &mut self,
5553        attempts: Vec<(AgentSpec, String)>,
5554        node: &'static str,
5555        round: Option<usize>,
5556        restore: Option<(&Path, &str)>,
5557        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5558    ) -> (SeatJob, SeatState, AgentOutcome) {
5559        let run_id = self.state.id.clone();
5560        let prompts = self.state.config.prompts.clone();
5561        let cache = self.state.config.cache_dir();
5562        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5563        let mut last = None;
5564        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5565            let seat = self.seat(&key, &spec.id);
5566            let mut job = build(spec.clone(), seat);
5567            if i > 0 {
5568                job.stem = format!("{}-{}", job.stem, spec.id);
5569            }
5570            let ctx = WaveCtx {
5571                carry_seats: false,
5572                pauses: Some((&self.pause, &self.interrupt)),
5573                run: &run_id,
5574                node,
5575                prompts: &prompts,
5576                cache: cache.as_deref(),
5577                round,
5578            };
5579            let (seat, out) =
5580                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5581            // A resource block never advances the chain: the chosen agent
5582            // was not asked, so it must be the one asked again on resume.
5583            let advances = match &out {
5584                AgentOutcome::Ok(o) => agent::output_advances(o),
5585                AgentOutcome::ResourceBlocked(_) => false,
5586                _ => true,
5587            };
5588            if let Some(next) = ids.get(i + 1)
5589                && advances
5590            {
5591                self.state.seats.insert(seat.key.clone(), seat.clone());
5592                let class =
5593                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5594                let reason = match &out {
5595                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5596                    other => fail_reason(other),
5597                };
5598                // A failed agent's edits must not reach the next one, and a
5599                // restore that did not work ends the chain: the call fails
5600                // and the caller's own (confirmed) restore decides.
5601                if let Some((wt, rev)) = restore
5602                    && let Err(e) = self.graft_restore(wt, rev).await
5603                {
5604                    let why =
5605                        format!("could not restore the worktree before the next fixer: {e:#}");
5606                    tracing::warn!("{why}");
5607                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5608                        AgentOutcome::ResourceBlocked(why)
5609                    } else {
5610                        AgentOutcome::Failed(why)
5611                    };
5612                    last = Some((job, seat, out));
5613                    break;
5614                }
5615                record_handover(
5616                    &mut self.state,
5617                    node,
5618                    &seat.key,
5619                    &spec.id,
5620                    next,
5621                    &class,
5622                    &reason,
5623                );
5624                continue;
5625            }
5626            last = Some((job, seat, out));
5627            break;
5628        }
5629        last.expect("the fixer chain always has an entry")
5630    }
5631
5632    /// The one repair a verification-only failure earns (see
5633    /// [`crate::recovery`]), before the first review round of a run that took
5634    /// a branch over.
5635    ///
5636    /// Saved as started *before* the fixer is called, so a resume never
5637    /// repairs twice; a start found on re-entry is rolled back to the commit
5638    /// it began from. A failed or unusable fixer leaves the tree as it was and
5639    /// the review goes on over it, so the first round's own verification
5640    /// reports the failure again. The failure is never carried into a round as
5641    /// a result: verification is re-run on whatever commit this leaves.
5642    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5643        use crate::recovery::{Mode, Repair};
5644        if !self.state.reviews.is_empty() {
5645            return Ok(());
5646        }
5647        let Some(mut rec) = self.state.recovery.clone() else {
5648            return Ok(());
5649        };
5650        if rec.mode != Mode::VerifyRepair {
5651            return Ok(());
5652        }
5653        let wt = winner.worktree.clone();
5654        let mut head = git::rev_parse(&wt, "HEAD").await?;
5655        match rec.repair.clone() {
5656            Repair::Done { .. } => return Ok(()),
5657            Repair::WaitingForRestore { before } => {
5658                self.graft_restore(&wt, &before).await?;
5659                head = before;
5660                rec.repair = Repair::NotStarted;
5661                self.state.recovery = Some(rec.clone());
5662                self.state.save()?;
5663            }
5664            Repair::Started { before } => {
5665                // Interrupted: the unfinished edits are not trusted.
5666                // An unconfirmed rollback blocks the run with `Started` kept,
5667                // so a resume retries it; the tree is never reviewed as is.
5668                if let Err(e) = self.graft_restore(&wt, &before).await {
5669                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5670                        return Err(e);
5671                    }
5672                    return self
5673                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5674                        .await;
5675                }
5676                rec.repair = Repair::Done {
5677                    before: before.clone(),
5678                    after: before,
5679                    committed: false,
5680                    failed: Some("interrupted; rolled back".to_owned()),
5681                };
5682                self.state
5683                    .event("recovery", "the repair was interrupted and rolled back");
5684                self.state.recovery = Some(rec);
5685                self.state.save()?;
5686                return Ok(());
5687            }
5688            Repair::NotStarted => {}
5689        }
5690        if crate::recovery::recheck(&mut rec, &head) {
5691            self.state.event("recovery", rec.summary());
5692            self.state.recovery = Some(rec);
5693            self.state.save()?;
5694            return Ok(());
5695        }
5696        rec.repair = Repair::Started {
5697            before: head.clone(),
5698        };
5699        self.state.recovery = Some(rec.clone());
5700        self.state.save()?;
5701
5702        let language = self.state.config.graph.language.clone();
5703        let sessions = self.state.config.graph.sessions;
5704        let artifacts = agent::artifacts_dir(&self.state.dir());
5705        let fix_prompt = prompt::recovery_fix(
5706            &self.state.instruction,
5707            &rec,
5708            self.state.config.graph.review_rounds,
5709            &language,
5710        );
5711        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5712        self.state.event(
5713            "recovery",
5714            format!(
5715                "repair: the fixer starts from the failed verification on {}",
5716                short(&head)
5717            ),
5718        );
5719        let (_job, seat, out) = self
5720            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5721                prompt: fix_prompt.clone(),
5722                spec,
5723                seat,
5724                cwd: wt.clone(),
5725                timeout,
5726                allow_write: true,
5727                sessions,
5728                artifacts: artifacts.clone(),
5729                stem: "fix-recovery".to_owned(),
5730                handover: None,
5731            })
5732            .await;
5733        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5734            rec.repair = Repair::WaitingForRestore { before: head };
5735            self.state.recovery = Some(rec);
5736            return self.stop_for_resource();
5737        }
5738        let failed = match &out {
5739            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5740            AgentOutcome::Quota(o) => {
5741                self.state.quota.push(QuotaLoss {
5742                    seat: seat.key.clone(),
5743                    node: "fix".to_owned(),
5744                    at: Timestamp::now(),
5745                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5746                });
5747                Some("rate limited (quota); the fixer could not run".to_owned())
5748            }
5749            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5750            AgentOutcome::Failed(e) => Some(e.clone()),
5751        };
5752        self.state.seats.insert(seat.key.clone(), seat);
5753        let coordination = self.coordinate(&wt).await?;
5754        let failed = match failed {
5755            // The agent's partial edits are not kept.
5756            Some(why) => {
5757                if let Err(e) = git::restore_to(&wt, &head).await {
5758                    return self
5759                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5760                        .await;
5761                }
5762                Some(why)
5763            }
5764            None => {
5765                if let Ok(r) = git::rescue_commit(
5766                    &wt,
5767                    "magi: recovery repair of the failed verification (uncommitted work)",
5768                )
5769                .await
5770                {
5771                    self.state.note_withheld("fix", &r.withheld);
5772                }
5773                None
5774            }
5775        };
5776        let after = git::rev_parse(&wt, "HEAD").await?;
5777        drop(coordination);
5778        self.state.event(
5779            "recovery",
5780            match &failed {
5781                Some(why) => format!("repair: nothing kept ({why})"),
5782                None if after != head => {
5783                    format!("repair: committed {}", short(&after))
5784                }
5785                None => "repair: the fixer made no commit".to_owned(),
5786            },
5787        );
5788        rec.repair = Repair::Done {
5789            before: head.clone(),
5790            committed: after != head,
5791            after,
5792            failed,
5793        };
5794        self.state.recovery = Some(rec);
5795        self.state.save()?;
5796        Ok(())
5797    }
5798
5799    /// A repair whose rollback could not be confirmed: the run is blocked and
5800    /// resumable, and the unconfirmed state is never reviewed.
5801    async fn stop_recovery(
5802        &mut self,
5803        mut rec: crate::recovery::RecoveryRecord,
5804        head: &str,
5805        why: String,
5806    ) -> Result<()> {
5807        // `Started` stays recorded: a resume retries the restore.
5808        rec.repair = crate::recovery::Repair::Started {
5809            before: head.to_owned(),
5810        };
5811        self.state.recovery = Some(rec);
5812        self.state
5813            .event("recovery", format!("repair could not be undone: {why}"));
5814        self.state.status = RunStatus::Blocked;
5815        self.state.save()?;
5816        bail!("recovery repair could not be rolled back: {why}")
5817    }
5818
5819    async fn review_loop(&mut self) -> Result<()> {
5820        // A base that would not rebase is a person's decision, not a review
5821        // round: nothing here would change the answer, and reviewers and a
5822        // fixer would be spending real budget on a tree that cannot land
5823        // regardless of what they find.
5824        if self
5825            .state
5826            .base_sync
5827            .as_ref()
5828            .is_some_and(|s| s.conflict.is_some())
5829        {
5830            return Ok(());
5831        }
5832        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5833        // agent files with `magi task add` name the run that paid for it. The
5834        // prompt overlay is cloned alongside it because the waves borrow it
5835        // while `self` is mutably borrowed by the node's own bookkeeping.
5836        let run_id = self.state.id.clone();
5837        let prompts = self.state.config.prompts.clone();
5838        let Some(winner) = self.state.winner().cloned() else {
5839            return Ok(());
5840        };
5841        let max_rounds = self.state.config.graph.review_rounds;
5842        // A clean round, an exhausted round budget, or a stalled tree (see
5843        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5844        // they are recorded — recomputed here, not read off `status`, so a
5845        // reentry into a run that already stopped restates the identical
5846        // verdict instead of silently handing back whatever an earlier node
5847        // in this same walk clobbered `status` to (a solo-candidate
5848        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5849        // below runs an empty range once the budget is spent, and would
5850        // otherwise fall through without touching `status` at all.
5851        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5852            // A reentry after a crash between the last round's save and
5853            // `stop_reviewing` reaches the hand-off here, not there.
5854            if status == RunStatus::Gating {
5855                self.record_contested_handoff();
5856            }
5857            self.state.status = status;
5858            self.state.save()?;
5859            return Ok(());
5860        }
5861        self.state.status = RunStatus::Reviewing;
5862        // A last recorded round whose own verification never resolved
5863        // (`ResourceBlocked` — the shared build cache, not the patch) is
5864        // never a concluded round, whatever the round budget says: starting
5865        // a fresh round on top of it would spend a whole new reviewer wave
5866        // re-reading an unchanged patch instead of just retrying the one
5867        // check that actually needs it, and once the budget is spent the
5868        // loop below has nothing left to do at all (its range is empty).
5869        // Retry that check directly instead, exactly the same retry
5870        // `stop_reviewing` already does for its own catch-up case.
5871        if self
5872            .state
5873            .reviews
5874            .last()
5875            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5876        {
5877            let shell = self.state.config.shell();
5878            return self
5879                .stop_reviewing(
5880                    "the last round's own verification never resolved",
5881                    &shell,
5882                    &winner.worktree,
5883                )
5884                .await;
5885        }
5886
5887        self.recovery_repair(&winner).await?;
5888        if self.resource_halt()? {
5889            return Ok(());
5890        }
5891
5892        let repo = self.state.repo.clone();
5893        let root = self.state.worktree_root();
5894        let language = self.state.config.graph.language.clone();
5895        let sessions = self.state.config.graph.sessions;
5896        let artifacts = self.merge_repair_artifacts();
5897        let base = self.landing_base();
5898        let base_short = short(&base);
5899        let reviewers = self.roles.reviewers.clone();
5900        let shell = self.state.config.shell();
5901
5902        for round in (self.state.reviews.len() + 1)..=max_rounds {
5903            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5904            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5905            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5906            // The prior round's own record, already persisted — never a
5907            // hand-carried variable of just its failing output: that is
5908            // exactly what let a round's e2e result drift out of sync with
5909            // which commit it was actually about (see `SCHEMA`'s doc for
5910            // schema 8). Judged against `head`, the commit reviewers are
5911            // about to look at now, so the summary always reads as "an
5912            // earlier head" here — this round's own patch has not been
5913            // checked yet.
5914            let prev_verification = self
5915                .state
5916                .reviews
5917                .last()
5918                .and_then(|r| r.verification_summary(&head));
5919            // Only the first round of a run that took a branch over: later
5920            // rounds have this run's own history.
5921            let prior_section = match (&self.state.recovery, round) {
5922                (Some(rec), 1) if self.state.reviews.is_empty() => {
5923                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5924                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5925                            Some(crate::recovery::head_of(
5926                                &git::diff(&winner.worktree, reviewed, "HEAD")
5927                                    .await
5928                                    .unwrap_or_default(),
5929                                crate::recovery::SINCE_DIFF_MAX,
5930                            ))
5931                        }
5932                        _ => None,
5933                    };
5934                    Some(prompt::recovery_review(rec, since.as_deref()))
5935                }
5936                _ => None,
5937            };
5938
5939            // Each reviewer gets its own detached checkout of exactly this
5940            // commit: nobody can perturb the winner's tree, and the fixer can
5941            // keep working without racing a reviewer.
5942            let mut jobs = Vec::new();
5943            let coordination = self.coordinate(&repo).await?;
5944            for (r, spec) in reviewers.iter().cloned().enumerate() {
5945                let wt = root.join(format!("review-{}", r + 1));
5946                // `clean -fdx` in a reset wipes setup's products, so every
5947                // round sets the seat up again.
5948                if wt.exists() {
5949                    git::reset_detached(&wt, &head).await?;
5950                } else {
5951                    git::worktree_add_detached(&repo, &wt, &head).await?;
5952                }
5953                self.setup_seat_worktree(&repo, &wt, false).await?;
5954                let seat_key = format!("review-{}", r + 1);
5955                // The seat starts the round on whoever answered it last, not
5956                // on the agent the spec names, so a failure is not re-paid.
5957                let spec = pick_start_spec(
5958                    &self.roles.reviewer_roster,
5959                    spec,
5960                    self.state.seat_history.get(&seat_key),
5961                );
5962                let seat = self.seat(&seat_key, &spec.id);
5963                jobs.push(SeatJob {
5964                    prompt: prompt::review(&prompt::ReviewCtx {
5965                        instruction: &self.state.instruction,
5966                        branch: &winner.branch,
5967                        base_short: &base_short,
5968                        stat: &stat,
5969                        patch: &patch,
5970                        verification: prev_verification.as_ref(),
5971                        prior: prior_section.as_deref(),
5972                        reviewers: reviewers.len(),
5973                        round,
5974                        rounds: max_rounds,
5975                        // A review-only run has no rankings, so nothing
5976                        // competed for this patch and the reviewer is told so.
5977                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5978                        lens: Lens::for_seat(r),
5979                        language: &language,
5980                        github_language: &self.state.config.graph.github_language,
5981                    }),
5982                    spec,
5983                    seat,
5984                    cwd: wt,
5985                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5986                    allow_write: false,
5987                    sessions,
5988                    artifacts: artifacts.clone(),
5989                    stem: format!("review-{round}-{}", r + 1),
5990                    handover: None,
5991                });
5992            }
5993
5994            drop(coordination);
5995            self.state.event(
5996                "review",
5997                format!(
5998                    "round {round}: {} reviewers on {}",
5999                    jobs.len(),
6000                    short(&head)
6001                ),
6002            );
6003            let mut quota_losses = Vec::new();
6004            let review_retries = self.state.config.graph.retries;
6005            let review_cache = self.state.config.cache_dir();
6006            let ctx = WaveCtx {
6007                carry_seats: true,
6008                pauses: Some((&self.pause, &self.interrupt)),
6009                run: &run_id,
6010                node: "review",
6011                prompts: &prompts,
6012                cache: review_cache.as_deref(),
6013                round: Some(round),
6014            };
6015            let results = ask_json_wave::<Review>(
6016                jobs,
6017                Arc::clone(&self.sem),
6018                review_retries,
6019                &self.roles.reviewer_roster,
6020                &ctx,
6021                &mut quota_losses,
6022                &mut self.state,
6023                &|_: &Review| Ok(()),
6024            )
6025            .await;
6026            if self.state.resource_block.is_some() {
6027                return self.stop_for_resource();
6028            }
6029            // Counted before the move below: how many of *this* round's
6030            // reviewer seats were lost to their own rate limit, as opposed to
6031            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6032            let round_quota_missing = quota_losses.len();
6033            self.state.quota.extend(quota_losses);
6034
6035            let mut records = Vec::new();
6036            let mut all_findings = Vec::new();
6037            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6038                let agent_id = seat.agent.clone();
6039                self.state.seats.insert(seat.key.clone(), seat);
6040                let mut record = ReviewRecord {
6041                    reviewer: r + 1,
6042                    agent: agent_id,
6043                    summary: String::new(),
6044                    findings: Vec::new(),
6045                    vote: None,
6046                    failed: None,
6047                    duration_ms: 0,
6048                    // Set for both outcomes: `failed: Some(_)` with
6049                    // `attempts > 0` is a seat every retry still lost, not a
6050                    // recovered one — only `failed: None` with `attempts > 0`
6051                    // reads as "answered after a nudge" (see this field's own
6052                    // doc).
6053                    attempts,
6054                };
6055                match res {
6056                    Ok((review, out)) => {
6057                        // Sanitized here, at the point every other piece of
6058                        // agent prose in this file is (candidate summaries,
6059                        // deliberation turns, vote reasons): a reviewer's own
6060                        // words are the one thing about it that could name
6061                        // it, and reconsideration below broadcasts this same
6062                        // summary and these same findings to every other
6063                        // seat on the panel.
6064                        record.summary =
6065                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6066                        record.vote = Some(review.vote);
6067                        record.duration_ms = out.duration_ms;
6068                        for (n, mut f) in review.findings.into_iter().enumerate() {
6069                            // ids are magi's, never the agent's: the fixer's
6070                            // adoption report is keyed by them.
6071                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6072                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6073                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6074                            // `file` is agent-supplied prose too, never
6075                            // checked against the real tree — the same
6076                            // exposure `title`/`detail` above have, just in
6077                            // a field easy to forget because it looks like a
6078                            // path rather than free text.
6079                            f.file = f
6080                                .file
6081                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6082                            all_findings.push(f.clone());
6083                            record.findings.push(f);
6084                        }
6085                        self.state.event(
6086                            "review",
6087                            format!(
6088                                "round {round}: reviewer {} voted {} with {} finding(s)",
6089                                r + 1,
6090                                review.vote.label(),
6091                                record.findings.len()
6092                            ),
6093                        );
6094                    }
6095                    Err(e) => {
6096                        record.failed = Some(e.to_string());
6097                        self.state.event(
6098                            "review",
6099                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6100                        );
6101                    }
6102                }
6103                records.push(record);
6104            }
6105
6106            // Tally the round's votes and, if they split, spend the one
6107            // round of reconsideration the split -> deliberate -> revote
6108            // shape `judge`/`vote` use for the panel, sized down to what a
6109            // read-only review round can afford: one round, and a revote
6110            // rather than an argument, because the panel already wrote its
6111            // reasoning down as findings the first time around.
6112            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6113            let vote_split =
6114                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6115            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6116            if vote_split {
6117                self.state.event(
6118                    "review",
6119                    format!(
6120                        "round {round}: votes split ({}) — one round of reconsideration",
6121                        initial_votes
6122                            .iter()
6123                            .map(|v| v.label())
6124                            .collect::<Vec<_>>()
6125                            .join(", ")
6126                    ),
6127                );
6128                // Seats read every seat's findings and votes, still numbered
6129                // and never named — the same anonymity `review` itself keeps.
6130                let panel: Vec<ReviewSeatReport<'_>> = records
6131                    .iter()
6132                    .filter_map(|r| {
6133                        r.vote.map(|vote| ReviewSeatReport {
6134                            reviewer: r.reviewer,
6135                            vote,
6136                            summary: &r.summary,
6137                            findings: &r.findings,
6138                        })
6139                    })
6140                    .collect();
6141
6142                let mut jobs = Vec::new();
6143                let mut seats_at = Vec::new();
6144                for (r, spec) in reviewers.iter().cloned().enumerate() {
6145                    // A seat with no initial vote has nothing to reconsider
6146                    // from and stays absent, the same as it stayed absent
6147                    // from `panel` above.
6148                    if records[r].vote.is_none() {
6149                        continue;
6150                    }
6151                    let wt = root.join(format!("review-{}", r + 1));
6152                    let seat_key = format!("review-{}", r + 1);
6153                    let spec = self.occupant(&seat_key, spec);
6154                    let seat = self.seat(&seat_key, &spec.id);
6155                    // A seat with no live session has already forgotten the
6156                    // initial review's prompt — restate the patch it is
6157                    // voting on, the same as `deliberate`/`vote` do for a
6158                    // judge in the same position.
6159                    // The panel already carries this seat's own review and
6160                    // vote, so restating the patch makes the prompt whole for
6161                    // a seat handed to another agent.
6162                    let build = |with_patch: bool| {
6163                        prompt::review_reconsider(&ReviewReconsiderCtx {
6164                            instruction: &self.state.instruction,
6165                            reviewer: r + 1,
6166                            lens: Lens::for_seat(r),
6167                            panel: &panel,
6168                            patch: with_patch.then_some(ReviewPatch {
6169                                branch: &winner.branch,
6170                                base_short: &base_short,
6171                                stat: &stat,
6172                                patch: &patch,
6173                            }),
6174                            round,
6175                            rounds: max_rounds,
6176                            language: &language,
6177                        })
6178                    };
6179                    let full = build(true);
6180                    let prompt = if has_context(&spec, &seat, sessions) {
6181                        build(false)
6182                    } else {
6183                        full.clone()
6184                    };
6185                    jobs.push(SeatJob {
6186                        prompt,
6187                        spec,
6188                        seat,
6189                        cwd: wt,
6190                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6191                        allow_write: false,
6192                        sessions,
6193                        artifacts: artifacts.clone(),
6194                        stem: format!("review-{round}-reconsider-{}", r + 1),
6195                        handover: Some(full),
6196                    });
6197                    seats_at.push(r);
6198                }
6199
6200                let mut recon_quota_losses = Vec::new();
6201                let recon_cache = self.state.config.cache_dir();
6202                let recon_ctx = WaveCtx {
6203                    carry_seats: true,
6204                    pauses: Some((&self.pause, &self.interrupt)),
6205                    run: &run_id,
6206                    node: "review",
6207                    prompts: &prompts,
6208                    cache: recon_cache.as_deref(),
6209                    round: Some(round),
6210                };
6211                let recon_results = ask_json_wave::<ReviewRevote>(
6212                    jobs,
6213                    Arc::clone(&self.sem),
6214                    review_retries,
6215                    &self.roles.reviewer_roster,
6216                    &recon_ctx,
6217                    &mut recon_quota_losses,
6218                    &mut self.state,
6219                    &|_: &ReviewRevote| Ok(()),
6220                )
6221                .await;
6222                if self.state.resource_block.is_some() {
6223                    return self.stop_for_resource();
6224                }
6225                self.state.quota.extend(recon_quota_losses);
6226
6227                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6228                    let agent_id = seat.agent.clone();
6229                    self.state.seats.insert(seat.key.clone(), seat);
6230                    let mut rec = ReviewRevoteRecord {
6231                        reviewer: r + 1,
6232                        agent: agent_id,
6233                        vote: None,
6234                        reason: String::new(),
6235                        failed: None,
6236                    };
6237                    match res {
6238                        Ok((rv, _)) => {
6239                            rec.vote = Some(rv.vote);
6240                            rec.reason =
6241                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6242                            self.state.event(
6243                                "review",
6244                                format!(
6245                                    "round {round}: reviewer {} revoted {}",
6246                                    r + 1,
6247                                    rv.vote.label()
6248                                ),
6249                            );
6250                        }
6251                        Err(e) => {
6252                            rec.failed = Some(e.to_string());
6253                            self.state.event(
6254                                "review",
6255                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6256                            );
6257                        }
6258                    }
6259                    reconsideration.push(rec);
6260                }
6261            } else if initial_votes.len() > 1 {
6262                self.state.event(
6263                    "review",
6264                    format!(
6265                        "round {round}: votes agreed ({}) — no reconsideration",
6266                        initial_votes[0].label()
6267                    ),
6268                );
6269            }
6270
6271            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6272            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6273            // A round that already has a blocking finding and a round left to
6274            // try is going back to the fixer no matter what `verify.e2e`
6275            // says, so running it first only spends the loop's slowest step
6276            // (minutes, for a Rust repo's full test suite) on a head about
6277            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6278            // runs once a round has no blocking findings left (see
6279            // `round_is_clean`, which a deferred — empty — `e2e` can never
6280            // satisfy since `blocking` is nonzero whenever this branch is
6281            // taken), and `stop_reviewing` forces a real run before it will
6282            // ever read a deferred round as green.
6283            let defer_e2e =
6284                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6285            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6286                let reason =
6287                    format!("{blocking} blocking finding(s) already required a fix this round");
6288                self.state.event(
6289                    "verify",
6290                    format!(
6291                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6292                         {}); it will run once a round has none left",
6293                        short(&head)
6294                    ),
6295                );
6296                (Vec::new(), false, true, Some(reason))
6297            } else {
6298                let e2e_commands = self.state.config.verify.e2e.clone();
6299                let cache_dir = self.state.config.cache_dir();
6300                let context = format!("round {round}");
6301                let wait = self.repository_wait();
6302                let (e2e, verify_retried) = with_cache_lease(
6303                    &mut self.state,
6304                    cache_dir.as_deref(),
6305                    "e2e",
6306                    "e2e",
6307                    &winner.worktree,
6308                    &head,
6309                    verify_timeout,
6310                    &context,
6311                    &wait,
6312                    |state, budget| {
6313                        let shell = shell.clone();
6314                        let e2e_commands = e2e_commands.clone();
6315                        let worktree = winner.worktree.clone();
6316                        let context = context.clone();
6317                        async move {
6318                            run_e2e_with_retry(
6319                                state,
6320                                &shell,
6321                                &e2e_commands,
6322                                &worktree,
6323                                budget,
6324                                &context,
6325                            )
6326                            .await
6327                        }
6328                    },
6329                )
6330                .await?;
6331                (e2e, verify_retried, false, None)
6332            };
6333
6334            let expected = records.len();
6335            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6336            let incomplete = answered < expected;
6337            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6338            let policy = self.state.config.graph.incomplete_review;
6339            let clean = round_is_clean(
6340                blocking,
6341                e2e_ok,
6342                answered,
6343                expected,
6344                round_quota_missing,
6345                policy,
6346            );
6347
6348            let mut round_record = ReviewRound {
6349                round,
6350                head: head.clone(),
6351                verified_head: None,
6352                verified_at: None,
6353                reviews: records,
6354                e2e,
6355                verify_retried,
6356                e2e_deferred,
6357                e2e_defer_reason,
6358                fix: None,
6359                blocking,
6360                answered,
6361                expected,
6362                clean,
6363                progressed: false,
6364                vote_split,
6365                reconsideration,
6366                verdict: None,
6367            };
6368            // The final vote per seat is its revote where reconsideration
6369            // ran and answered, its initial vote otherwise — the same
6370            // fallback `tally` uses for a judge whose private vote failed.
6371            round_record.verdict = ReviewVote::worst(
6372                round_record
6373                    .final_votes()
6374                    .into_iter()
6375                    .map(|(_, _, vote)| vote),
6376            );
6377            // Which commit and when magi actually attempted to check —
6378            // known the moment a command was dispatched against `head`,
6379            // whether or not it finished: a resource-blocked attempt still
6380            // targeted a specific commit at a specific time, and leaving
6381            // that unrecorded is exactly what made `verification_summary`
6382            // report a fresh attempt as "commit unknown ... recorded before
6383            // this was tracked", indistinguishable from a genuinely old,
6384            // untracked record. Only a deferred or unconfigured round never
6385            // ran at all and has nothing to record — see
6386            // `ReviewRound::verified_head`'s own doc.
6387            if !matches!(
6388                round_record.e2e_status(),
6389                E2eStatus::Deferred | E2eStatus::NotConfigured
6390            ) {
6391                round_record.verified_head = Some(head.clone());
6392                round_record.verified_at = Some(Timestamp::now());
6393            }
6394            let this_round_verification = round_record.verification_summary(&head);
6395
6396            if incomplete {
6397                let missing: Vec<String> = round_record
6398                    .reviews
6399                    .iter()
6400                    .filter(|r| r.failed.is_some())
6401                    .map(|r| format!("review-{}", r.reviewer))
6402                    .collect();
6403                self.state.event(
6404                    "review",
6405                    format!(
6406                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6407                        missing.join(", ")
6408                    ),
6409                );
6410            }
6411
6412            if clean {
6413                self.state.event(
6414                    "review",
6415                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6416                        format!(
6417                            "round {round}: clean (warn policy, incomplete panel) — no \
6418                             blocking findings from the seats that answered, verification green"
6419                        )
6420                    } else if incomplete {
6421                        format!(
6422                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6423                             quorum) — no blocking findings from the seats that answered, \
6424                             verification green",
6425                            expected - answered
6426                        )
6427                    } else {
6428                        format!("round {round}: clean — no blocking findings, verification green")
6429                    },
6430                );
6431                self.state.reviews.push(round_record);
6432                self.state.status = RunStatus::Gating;
6433                self.state.save()?;
6434                return Ok(());
6435            }
6436
6437            // Nothing was raised and verification passed, but not every seat
6438            // answered and `round_is_clean` still refused to call it clean —
6439            // either a seat is missing for a reason other than its own quota
6440            // (a crash, a timeout, unparsable output — worth another try), or
6441            // every seat that could have answered lost its quota and nobody
6442            // is left to decide on: re-review rather than send the fixer
6443            // after a round with nothing to fix.
6444            if incomplete && blocking == 0 && e2e_ok {
6445                self.state.reviews.push(round_record);
6446                self.state.save()?;
6447                if round == max_rounds {
6448                    self.state.status = RunStatus::Blocked;
6449                    self.state.event(
6450                        "review",
6451                        format!(
6452                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6453                             refusing to call it clean",
6454                            expected - answered
6455                        ),
6456                    );
6457                    return Ok(());
6458                }
6459                continue;
6460            }
6461
6462            // Nothing for the fixer to act on (`blocking == 0`) and the only
6463            // reason this round is not clean is that magi itself never got
6464            // a command to run — the shared build cache, not the patch (see
6465            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6466            // the fixer would invite a change to appease contention that has
6467            // nothing to do with the diff, and would leave this attempt
6468            // sitting in the next round's prompt as if it were about an
6469            // earlier, superseded commit rather than what it actually is:
6470            // the same head, still waiting to be checked. Wait for it the
6471            // same way the final round's own contention is already handled,
6472            // whatever round this happens to be.
6473            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6474                self.state.reviews.push(round_record);
6475                return self
6476                    .stop_reviewing(
6477                        "the round's own verification could not run",
6478                        &shell,
6479                        &winner.worktree,
6480                    )
6481                    .await;
6482            }
6483
6484            if round == max_rounds {
6485                self.state.reviews.push(round_record);
6486                return self
6487                    .stop_reviewing(
6488                        &format!(
6489                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6490                        ),
6491                        &shell,
6492                        &winner.worktree,
6493                    )
6494                    .await;
6495            }
6496
6497            // Fix. The winner's own implementer seat continues its conversation:
6498            // the competition is over, so context is pure benefit now.
6499            let blocking_findings: Vec<_> = all_findings
6500                .iter()
6501                .filter(|f| f.severity.blocks())
6502                .cloned()
6503                .collect();
6504            let fix_prompt = prompt::fix_in(
6505                &self.state.instruction,
6506                &blocking_findings,
6507                this_round_verification.as_ref(),
6508                round,
6509                max_rounds,
6510                &language,
6511                &self.state.config.graph.github_language,
6512            );
6513            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6514            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6515            let (job, seat, out) = self
6516                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6517                    prompt: fix_prompt.clone(),
6518                    spec,
6519                    seat,
6520                    cwd: winner.worktree.clone(),
6521                    timeout,
6522                    allow_write: true,
6523                    sessions,
6524                    artifacts: artifacts.clone(),
6525                    stem: format!("fix-{round}"),
6526                    handover: None,
6527                })
6528                .await;
6529            // Not asked: the round is not recorded (so a resume repeats it),
6530            // no fixer was handed over to, and the seat is left as it was.
6531            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6532                return self.stop_for_resource();
6533            }
6534            let agent_id = seat.agent.clone();
6535
6536            let mut fix = FixRecord {
6537                agent: agent_id,
6538                addressed: Vec::new(),
6539                rejected: Vec::new(),
6540                notes: String::new(),
6541                committed: false,
6542                failed: None,
6543                duration_ms: 0,
6544                continuation: None,
6545            };
6546            let mut continuation = ContinuationRecord::not_needed();
6547            let mut final_seat = seat.clone();
6548            match out {
6549                AgentOutcome::Ok(o) => {
6550                    fix.duration_ms = o.duration_ms;
6551                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6552                    // A parsed report standing next to a command this same
6553                    // reply's own CLI never confirmed the exit status of is
6554                    // not a resolved answer — the identical `CommandEvidence`
6555                    // `state.jobs` renders, read here instead of only on
6556                    // display, per the completion judgment and the shown
6557                    // record needing to agree.
6558                    let incomplete_reason = match &parsed {
6559                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6560                            "the reply parsed, but it reported a command whose own CLI never \
6561                             confirmed an exit status"
6562                                .to_owned(),
6563                        ),
6564                        Ok(_) => None,
6565                        Err(e) => Some(e.to_string()),
6566                    };
6567                    match incomplete_reason {
6568                        None => {
6569                            let report = parsed.expect("checked Ok above");
6570                            fix.addressed = report.addressed;
6571                            fix.rejected = report.rejected;
6572                            fix.notes =
6573                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6574                        }
6575                        Some(reason) => {
6576                            let (resumed_seat, resolved, failure, cont) = self
6577                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6578                                .await;
6579                            fix.duration_ms += cont.cumulative_wait_ms;
6580                            continuation = cont;
6581                            final_seat = resumed_seat;
6582                            match resolved {
6583                                Some(report) => {
6584                                    fix.addressed = report.addressed;
6585                                    fix.rejected = report.rejected;
6586                                    fix.notes = blind::sanitize_prose(
6587                                        &report.notes,
6588                                        &self.state.config.blind,
6589                                    );
6590                                }
6591                                None => fix.failed = failure,
6592                            }
6593                        }
6594                    }
6595                }
6596                // The CLI's raw error JSON is not a fix report to parse.
6597                AgentOutcome::Dropped(o) => {
6598                    fix.duration_ms = o.duration_ms;
6599                    let why = o
6600                        .dropped
6601                        .as_ref()
6602                        .map(|d| d.why.as_str())
6603                        .unwrap_or("the CLI ended the stream without delivering its answer");
6604                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6605                }
6606                AgentOutcome::Quota(o) => {
6607                    self.state.quota.push(QuotaLoss {
6608                        seat: final_seat.key.clone(),
6609                        node: "fix".to_owned(),
6610                        at: Timestamp::now(),
6611                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6612                    });
6613                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6614                }
6615                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6616            }
6617            fix.continuation = Some(continuation);
6618            self.state.seats.insert(final_seat.key.clone(), final_seat);
6619            let coordination = self.coordinate(&winner.worktree).await?;
6620            if let Ok(r) = git::rescue_commit(
6621                &winner.worktree,
6622                &format!("magi: review round {round} fixes (uncommitted work)"),
6623            )
6624            .await
6625            {
6626                self.state.note_withheld("fix", &r.withheld);
6627            }
6628            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6629            fix.committed = after != before;
6630            // Judged by what `git` says moved against base, never by the
6631            // fixer's own `addressed`/`rejected` count — see
6632            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6633            // `patch` snapshot above: swallowing this error would default
6634            // `diff_after` to empty, which almost always differs from a
6635            // non-empty `patch` and reads as "progressed" — exactly backwards
6636            // for a `git` failure the stagnation check cannot see through.
6637            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6638            drop(coordination);
6639            let progressed = diff_after != patch;
6640            let commit_note = if fix.committed {
6641                "committed"
6642            } else {
6643                "NO new commit"
6644            };
6645            let tree_note = if progressed {
6646                "changed vs base"
6647            } else {
6648                "unchanged vs base"
6649            };
6650            self.state.event(
6651                "fix",
6652                match &fix.failed {
6653                    // Distinct on purpose from "0 addressed, 0 rejected": the
6654                    // fixer's own diff still landed (blocking counts do keep
6655                    // falling round over round), only its adoption report did
6656                    // not come back, so this must never read like every
6657                    // finding was reviewed and declined.
6658                    Some(reason) => {
6659                        format!(
6660                            "round {round}: fixer's adoption report was lost ({reason}); \
6661                             {commit_note}, tree {tree_note}"
6662                        )
6663                    }
6664                    None => format!(
6665                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6666                         {tree_note}{}",
6667                        fix.addressed.len(),
6668                        fix.rejected.len(),
6669                        if continuation.outcome == ContinuationOutcome::Resumed {
6670                            format!(
6671                                " (adoption report recovered after {} continuation(s))",
6672                                continuation.attempts
6673                            )
6674                        } else {
6675                            String::new()
6676                        },
6677                    ),
6678                },
6679            );
6680            round_record.fix = Some(fix);
6681            round_record.progressed = progressed;
6682            self.state.reviews.push(round_record);
6683            self.state.save()?;
6684
6685            // The fixer's own report never came back this round, even after
6686            // `continue_fix_report`'s own budget was spent on it — not an
6687            // ordinary "no report" (dropped stream, quota, plain failure),
6688            // which already reads that way and is left to the existing round
6689            // budget. Stopping here, rather than opening another round, is
6690            // what keeps a next reviewer/fixer wave from ever being
6691            // dispatched onto `winner.worktree` while whatever the seat's
6692            // last call may still have running there is unaccounted for: no
6693            // process liveness check exists (and none is being added — see
6694            // AGENTS.md/this task's own scope), so the only way to honour
6695            // "nothing starts before a valid report returns" is to not start
6696            // anything further on this worktree from this run at all.
6697            if matches!(
6698                continuation.outcome,
6699                ContinuationOutcome::Exhausted
6700                    | ContinuationOutcome::QuotaLost
6701                    | ContinuationOutcome::NoSession
6702            ) {
6703                return self
6704                    .stop_reviewing(
6705                        "the fixer's adoption report never came back, even after resuming its \
6706                         own seat; refusing to start another round against the same worktree \
6707                         while that is unresolved",
6708                        &shell,
6709                        &winner.worktree,
6710                    )
6711                    .await;
6712            }
6713
6714            let streak = self
6715                .state
6716                .reviews
6717                .iter()
6718                .rev()
6719                .take_while(|r| !r.progressed)
6720                .count();
6721            if streak >= STAGNANT_LIMIT {
6722                return self
6723                    .stop_reviewing(
6724                        &format!(
6725                            "the tree has not moved against base for {streak} round(s) in a row"
6726                        ),
6727                        &shell,
6728                        &winner.worktree,
6729                    )
6730                    .await;
6731            }
6732        }
6733        Ok(())
6734    }
6735
6736    /// Decide, from the last recorded round's own verification, whether
6737    /// stopping the review loop is a hand-off or a genuine block.
6738    ///
6739    /// Called once the loop has given up trying — the round budget is spent,
6740    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6741    /// findings still open, never while a round is still clean or the
6742    /// incomplete-panel case handled inline above. Gate and e2e are facts
6743    /// about the tree; a lingering review finding is an opinion, and this
6744    /// workload's own `magi stats` puts reviewer precision low enough
6745    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6746    /// must not by itself stand between a green, verified change and the
6747    /// human who decides what to do with it. A red e2e is not an opinion, so
6748    /// that case still blocks, with the failing command and a tail of its
6749    /// output recorded here rather than left in `run.json` for someone to go
6750    /// find.
6751    ///
6752    /// A round that deferred its own e2e (see [`Config::graph`]'s
6753    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6754    /// only because nothing ran, and treating an empty list as a passing one
6755    /// here is exactly the "deferred painted green" bug this function exists
6756    /// to not have. When the last round's own verification never resolved —
6757    /// deferred on purpose, or a real attempt the shared build cache blocked
6758    /// — this makes (or retries) the real run, on the actual worktree this
6759    /// loop is about to stop touching, before deciding anything. A
6760    /// resource-blocked attempt is likewise never read as either green or
6761    /// red: it is evidence about the machine, not the patch (see
6762    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6763    /// blocked cache leaves this call without deciding rather than guessing
6764    /// — the caller retries on a later reentry.
6765    /// Record, once, that the review loop handed off over a blocking finding
6766    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6767    /// `land` asks the owner even with `land_approval` off. Called from every
6768    /// path that concludes `Gating`; a reentry keeps the first record.
6769    fn record_contested_handoff(&mut self) {
6770        if self.state.contested_handoff.is_some() {
6771            return;
6772        }
6773        let Some(contested) = self
6774            .state
6775            .reviews
6776            .last()
6777            .and_then(ReviewRound::contested_handoff)
6778        else {
6779            return;
6780        };
6781        self.state.event(
6782            "review",
6783            format!(
6784                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6785                 wait for the owner's approval",
6786                contested.findings.len(),
6787                contested.rejecters.len()
6788            ),
6789        );
6790        self.state.contested_handoff = Some(contested);
6791    }
6792
6793    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6794        let round_idx = self.state.reviews.len() - 1;
6795        // A deferred round and a resource-blocked one are the same shape
6796        // here: neither has a real result yet, and both get one more
6797        // attempt. Read off `e2e_status` — the single source for this —
6798        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6799        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6800        // still retries instead of being read as a settled result the
6801        // instant it stops being empty.
6802        let needs_catchup_run = matches!(
6803            self.state.reviews[round_idx].e2e_status(),
6804            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6805        );
6806        if needs_catchup_run {
6807            let round = self.state.reviews[round_idx].round;
6808            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6809            let commands = self.state.config.verify.e2e.clone();
6810            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6811            let cache_dir = self.state.config.cache_dir();
6812            let context = format!(
6813                "round {round}: verification unresolved, catching up before the final decision"
6814            );
6815            let wait = self.repository_wait();
6816            let (outcomes, verify_retried) = with_cache_lease(
6817                &mut self.state,
6818                cache_dir.as_deref(),
6819                "e2e",
6820                "e2e",
6821                worktree,
6822                &attempted_head,
6823                timeout,
6824                &context,
6825                &wait,
6826                |state, budget| {
6827                    let shell = shell.to_vec();
6828                    let commands = commands.clone();
6829                    let context = context.clone();
6830                    async move {
6831                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6832                            .await
6833                    }
6834                },
6835            )
6836            .await?;
6837            let last = &mut self.state.reviews[round_idx];
6838            last.e2e = outcomes;
6839            last.verify_retried = verify_retried;
6840            // Always the commit and time this attempt actually targeted,
6841            // whether or not it happens to equal the reviewed `head` and
6842            // whether or not a command finished — see
6843            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6844            // attempt is recorded too, so a later reader sees "attempted
6845            // again at T2" rather than silence.
6846            last.verified_head = Some(attempted_head);
6847            last.verified_at = Some(Timestamp::now());
6848            if verify_inconclusive(&last.e2e) {
6849                // Still not a real result: `e2e_deferred` is left exactly
6850                // as it was, so `needs_catchup_run` above reads
6851                // `ResourceBlocked` (via `e2e_status`, which checks
6852                // `resource_blocked` before `e2e_deferred`) and retries
6853                // again on the next reentry, rather than recording
6854                // contention as a red e2e and blocking the run on it.
6855                self.state.save()?;
6856                return Ok(());
6857            }
6858            last.e2e_deferred = false;
6859        }
6860        let last = &self.state.reviews[round_idx];
6861        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6862
6863        match last.e2e_status() {
6864            E2eStatus::Failed => {
6865                let red: Vec<String> = last
6866                    .e2e
6867                    .iter()
6868                    .filter(|o| !o.ok())
6869                    .map(|o| {
6870                        format!(
6871                            "`{}` -> {:?}\n{}",
6872                            o.command,
6873                            o.code,
6874                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6875                        )
6876                    })
6877                    .collect();
6878                self.state
6879                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6880                self.state.status = RunStatus::Blocked;
6881            }
6882            // `needs_catchup_run` above already retried once this call; if
6883            // it is still blocked, this is magi's own admission it could
6884            // not get a command to run, never a verdict on the patch — the
6885            // run is left exactly where a later reentry can retry again.
6886            E2eStatus::ResourceBlocked => {
6887                self.state.event(
6888                    "review",
6889                    format!(
6890                        "{why}; e2e could not run (shared build cache unavailable); not \
6891                         deciding yet"
6892                    ),
6893                );
6894            }
6895            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6896                self.state.event(
6897                    "review",
6898                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6899                );
6900                self.record_contested_handoff();
6901                self.state.status = RunStatus::Gating;
6902            }
6903        }
6904        self.state.save()?;
6905        Ok(())
6906    }
6907
6908    // ----------------------------------------------------------------- gate
6909
6910    async fn gate(&mut self) -> Result<()> {
6911        // Judged by the review record itself, not by `status`: a solo
6912        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6913        // reentry (see `judge`), and trusting it here is exactly how a run
6914        // that exhausted its review budget got gated and merged a second
6915        // time around. `review_conclusion` recomputes the review loop's own
6916        // verdict from the round records themselves — `Gating` for a clean
6917        // round or a hand-off (see `stop_reviewing`), anything else means the
6918        // loop is still going or genuinely blocked.
6919        // A base the winner could not be replayed onto is a decision, not a
6920        // round: there is no landing tree to gate. Read as its own record for
6921        // the same reason the review verdict is.
6922        if self.state.status == RunStatus::Failed
6923            || self
6924                .state
6925                .base_sync
6926                .as_ref()
6927                .is_some_and(|s| s.conflict.is_some())
6928            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6929                != Some(RunStatus::Gating)
6930        {
6931            return Ok(());
6932        }
6933        if self.state.gate_ran {
6934            // `review_loop` derives its conclusion from the clean review
6935            // record on every reentry and therefore puts a completed run back
6936            // in `Gating`. A recorded gate is a stronger, terminal fact:
6937            // retain its original command output (or lack of any, for a repo
6938            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6939            // doc) and restore `Blocked` on a real failure rather than
6940            // pretending the command is still running or running it a second
6941            // time. `gate_ran == false` remains the only shape — unattempted,
6942            // or a resource-blocked retry — that may still need to execute a
6943            // command.
6944            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6945                self.state.status = RunStatus::Blocked;
6946                self.state.save()?;
6947            }
6948            return Ok(());
6949        }
6950        let Some(winner) = self.state.winner().cloned() else {
6951            return Ok(());
6952        };
6953        self.state.status = RunStatus::Gating;
6954        let mut outcomes = self.run_gate(&winner).await?;
6955        loop {
6956            // A resource-blocked outcome means the gate command never actually
6957            // ran - the shared build cache could not be acquired or confirmed
6958            // fresh in time - which is evidence about the machine, not about
6959            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6960            // Recording it as a red gate would mark a run `Blocked` on nothing
6961            // but contention magi has already logged; leaving `self.state.gate`
6962            // empty and `self.state.gate_ran` false instead keeps the shape
6963            // this function already treats as "still needs to run" (see the
6964            // early-return above), so the next call retries the command
6965            // rather than concluding anything.
6966            if verify_inconclusive(&outcomes) {
6967                self.state.save()?;
6968                return Ok(());
6969            }
6970            if outcomes.iter().all(CommandOutcome::ok) {
6971                break;
6972            }
6973            match self.gate_fix_round(&winner, &outcomes).await? {
6974                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6975                GateFix::Stop => break,
6976                GateFix::Defer => {
6977                    self.state.save()?;
6978                    return Ok(());
6979                }
6980            }
6981        }
6982        let passed = outcomes.iter().all(CommandOutcome::ok);
6983        self.state.gate = outcomes;
6984        self.state.gate_ran = true;
6985        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6986        if !passed {
6987            self.state.status = RunStatus::Blocked;
6988            let spent = self.state.gate_fixes.len();
6989            self.state.event(
6990                "gate",
6991                if spent == 0 {
6992                    "gate failed; not merging".to_owned()
6993                } else {
6994                    format!("gate failed after {spent} gate-fix round(s); not merging")
6995                },
6996            );
6997        }
6998        self.state.save()?;
6999        Ok(())
7000    }
7001
7002    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7003    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7004    /// after review is clean and never on a candidate awaiting judging.
7005    ///
7006    /// Never fails the run: a non-zero exit or timeout is a warning and a
7007    /// recorded outcome, and the gate remains the single arbiter. Nothing
7008    /// configured means nothing happens - no event, no commit. `commit_all`
7009    /// commits any leftover change under the neutral identity and returns
7010    /// `false` when the tree is clean, so no empty commit is ever made.
7011    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7012        let commands = state.config.verify.pre_gate.clone();
7013        if commands.is_empty() {
7014            return;
7015        }
7016        let shell = state.config.shell();
7017        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7018        let (outcomes, _) = run_commands(
7019            state,
7020            "pre_gate",
7021            "pre_gate",
7022            0,
7023            &shell,
7024            &commands,
7025            &winner.worktree,
7026            timeout,
7027        )
7028        .await;
7029        for o in &outcomes {
7030            if !o.ok() {
7031                tracing::warn!(
7032                    "pre_gate `{}` failed ({:?}); the gate decides",
7033                    o.command,
7034                    o.code
7035                );
7036            }
7037            state.event(
7038                "pre_gate",
7039                format!(
7040                    "`{}` -> {}",
7041                    o.command,
7042                    if o.ok() {
7043                        "pass".to_owned()
7044                    } else {
7045                        format!(
7046                            "FAIL ({:?})\n{}",
7047                            o.code,
7048                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7049                        )
7050                    }
7051                ),
7052            );
7053        }
7054        state.pre_gate = outcomes;
7055        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7056            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7057                Ok(head) => {
7058                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7059                    state.pre_gate_commit = Some(head);
7060                }
7061                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7062            },
7063            Ok(false) => {}
7064            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7065        }
7066        if let Err(e) = state.save() {
7067            tracing::warn!("could not persist the pre_gate record: {e:#}");
7068        }
7069    }
7070
7071    /// Run `verify.gate` once against the winner's current tree, logging one
7072    /// event per command. Empty when nothing is configured.
7073    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7074        let shell = self.state.config.shell();
7075        let gate_commands = self.state.config.verify.gate.clone();
7076        let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7077        let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7078            Vec::new()
7079        } else {
7080            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7081            let cache_dir = self.state.config.cache_dir();
7082            let wait = self.repository_wait();
7083            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7084            let (outcomes, _) = with_cache_lease(
7085                &mut self.state,
7086                cache_dir.as_deref(),
7087                "gate",
7088                "gate",
7089                &winner.worktree,
7090                &head,
7091                timeout,
7092                "final gate",
7093                &wait,
7094                |state, budget| {
7095                    let shell = shell.clone();
7096                    let gate_commands = gate_commands.clone();
7097                    let winner = winner.clone();
7098                    async move {
7099                        Self::run_pre_gate(state, &winner).await;
7100                        let (outcomes, timed_out_pids) = run_commands(
7101                            state,
7102                            "gate",
7103                            "gate",
7104                            0,
7105                            &shell,
7106                            &gate_commands,
7107                            &winner.worktree,
7108                            budget,
7109                        )
7110                        .await;
7111                        (outcomes, false, timed_out_pids)
7112                    }
7113                },
7114            )
7115            .await?;
7116            outcomes
7117        };
7118        if outcomes.is_empty() {
7119            // Nothing configured to check — distinct from every other
7120            // silence in this run's event log, since an empty `gate` alone
7121            // no longer says whether the gate ran at all (see
7122            // `RunState::gate_ran`'s own doc).
7123            self.state
7124                .event("gate", "no gate commands configured; not tested");
7125        }
7126        for o in &outcomes {
7127            self.state.event(
7128                "gate",
7129                format!(
7130                    "`{}` -> {}",
7131                    o.command,
7132                    if o.ok() {
7133                        "pass".to_owned()
7134                    } else {
7135                        format!(
7136                            "FAIL ({:?})\n{}",
7137                            o.code,
7138                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7139                        )
7140                    }
7141                ),
7142            );
7143        }
7144        Ok(outcomes)
7145    }
7146
7147    /// One bounded fix round for a failing gate.
7148    ///
7149    /// The fixer is told the failure came from the gate itself, not from a
7150    /// reviewer, and is shown the failed commands, their exit codes and a tail
7151    /// of their output - whatever `[verify].gate` holds, nothing here knows
7152    /// what those commands run. Only a normal non-zero exit that printed
7153    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7154    /// command or a full disk says nothing about the code, and a fixer sent
7155    /// after it can only appease the machine. The round is judged by what git
7156    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7157    /// again before the gate does, so a fix cannot trade a green gate for a
7158    /// red e2e unnoticed.
7159    async fn gate_fix_round(
7160        &mut self,
7161        winner: &Candidate,
7162        outcomes: &[CommandOutcome],
7163    ) -> Result<GateFix> {
7164        let cap = self.state.config.graph.gate_fix_rounds;
7165        let spent = self.state.gate_fixes.len();
7166        if spent >= cap {
7167            if cap > 0 {
7168                self.state.event(
7169                    "gate",
7170                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7171                );
7172            }
7173            return Ok(GateFix::Stop);
7174        }
7175        if !gate_fixable(outcomes) {
7176            self.state.event(
7177                "gate",
7178                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7179                 command or similar); not spending a fix round on it",
7180            );
7181            return Ok(GateFix::Stop);
7182        }
7183        let min_free = self.state.config.disk.min_free_bytes;
7184        if min_free > 0 {
7185            match crate::disk::free_bytes(&winner.worktree) {
7186                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7187                Ok(free) => {
7188                    self.state.event(
7189                        "gate",
7190                        format!(
7191                            "only {free} bytes free ({min_free} required by `[disk] \
7192                             min_free_bytes`); not spending a fix round on a failure the disk \
7193                             may explain"
7194                        ),
7195                    );
7196                    return Ok(GateFix::Stop);
7197                }
7198                Err(e) => {
7199                    self.state.event(
7200                        "gate",
7201                        format!("free disk space could not be measured ({e:#}); no fix round"),
7202                    );
7203                    return Ok(GateFix::Stop);
7204                }
7205            }
7206        }
7207
7208        let attempt = spent + 1;
7209        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7210        let base = self.landing_base();
7211        let fix_prompt = prompt::gate_fix_in(
7212            &self.state.instruction,
7213            &failed,
7214            attempt,
7215            cap,
7216            &self.state.config.graph.language,
7217            &self.state.config.graph.github_language,
7218        );
7219        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7220        let sessions = self.state.config.graph.sessions;
7221        let artifacts = self.merge_repair_artifacts();
7222        self.state.event(
7223            "gate",
7224            format!("gate failed; gate-fix round {attempt} of {cap}"),
7225        );
7226        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7227        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7228        let (_, seat, out) = self
7229            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7230                prompt: fix_prompt.clone(),
7231                spec,
7232                seat,
7233                cwd: winner.worktree.clone(),
7234                timeout,
7235                allow_write: true,
7236                sessions,
7237                artifacts: artifacts.clone(),
7238                stem: format!("gate-fix-{attempt}"),
7239                handover: None,
7240            })
7241            .await;
7242        // The fixer was never asked: no round is spent (nothing is recorded)
7243        // and a later reentry asks the same agent again.
7244        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7245            return Ok(GateFix::Defer);
7246        }
7247        let mut record = GateFixRecord {
7248            agent: seat.agent.clone(),
7249            failed,
7250            notes: String::new(),
7251            committed: false,
7252            error: None,
7253        };
7254        match out {
7255            AgentOutcome::Ok(o) => {
7256                // A missing report is not a failed fix: the round is judged
7257                // by the tree below, and the report only carries prose.
7258                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7259                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7260                }
7261            }
7262            AgentOutcome::Dropped(_) => {
7263                record.error = Some("the CLI dropped the stream".to_owned());
7264            }
7265            AgentOutcome::Quota(o) => {
7266                self.state.quota.push(QuotaLoss {
7267                    seat: seat.key.clone(),
7268                    node: "gate-fix".to_owned(),
7269                    at: Timestamp::now(),
7270                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7271                });
7272                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7273            }
7274            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7275        }
7276        self.state.seats.insert(seat.key.clone(), seat);
7277        let coordination = self.coordinate(&winner.worktree).await?;
7278        if let Ok(r) = git::rescue_commit(
7279            &winner.worktree,
7280            &format!("magi: gate fix {attempt} (uncommitted work)"),
7281        )
7282        .await
7283        {
7284            self.state.note_withheld("gate-fix", &r.withheld);
7285        }
7286        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7287        record.committed = after != before;
7288        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7289        let note = record.error.clone();
7290        self.state.gate_fixes.push(record);
7291        self.state.save()?;
7292        if !changed {
7293            self.state.event(
7294                "gate",
7295                match note {
7296                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7297                    None => format!("gate-fix round {attempt}: the tree did not change"),
7298                },
7299            );
7300            return Ok(GateFix::Stop);
7301        }
7302        self.state.event(
7303            "gate",
7304            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7305        );
7306
7307        drop(coordination);
7308        let commands = self.state.config.verify.e2e.clone();
7309        if !commands.is_empty() {
7310            let shell = self.state.config.shell();
7311            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7312            let cache_dir = self.state.config.cache_dir();
7313            let context = format!("gate-fix round {attempt}");
7314            let wait = self.repository_wait();
7315            let (e2e, _) = with_cache_lease(
7316                &mut self.state,
7317                cache_dir.as_deref(),
7318                "e2e",
7319                "e2e",
7320                &winner.worktree,
7321                &after,
7322                timeout,
7323                &context,
7324                &wait,
7325                |state, budget| {
7326                    let shell = shell.clone();
7327                    let commands = commands.clone();
7328                    let context = context.clone();
7329                    let worktree = winner.worktree.clone();
7330                    async move {
7331                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7332                            .await
7333                    }
7334                },
7335            )
7336            .await?;
7337            if verify_inconclusive(&e2e) {
7338                return Ok(GateFix::Defer);
7339            }
7340            if e2e.iter().any(|o| !o.ok()) {
7341                self.state.event(
7342                    "gate",
7343                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7344                );
7345                return Ok(GateFix::Stop);
7346            }
7347        }
7348        Ok(GateFix::Retry)
7349    }
7350
7351    // ---------------------------------------------------------------- merge
7352
7353    /// One read-only rewrite by the summary's author; a text the gate still
7354    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7355    /// applies on their word or on silence. `None` means the run is parked on
7356    /// that question and nothing may be pushed or opened yet.
7357    async fn guarded_pr_message(
7358        &mut self,
7359        winner: &Candidate,
7360        facts: Option<&BranchFacts>,
7361        posting: bool,
7362    ) -> Result<Option<PrMessage>> {
7363        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7364        let head = match final_head.as_deref() {
7365            Some(h) => crate::pr_outcome::FinalHead::At(h),
7366            None => crate::pr_outcome::FinalHead::Unknown,
7367        };
7368        // Only a pull request that is about to be posted pays for a summary.
7369        let summarized = if posting {
7370            Some(self.summarize_change(winner, final_head.as_deref()).await)
7371        } else {
7372            None
7373        };
7374        // A busy shared resource is not a permanent cause: publishing an
7375        // "unavailable" description would stick (a later head move never
7376        // edits the body). Stop before anything is pushed or opened; the
7377        // run is `Blocked` and a resume asks the summary again.
7378        if self.state.resource_block.is_some() {
7379            self.state.event(
7380                "pr-summary",
7381                "a shared resource was busy; nothing was published, resume to summarize again",
7382            );
7383            return Ok(None);
7384        }
7385        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7386            anyhow::bail!(
7387                "the final commit moved during public description generation; retry before publishing"
7388            );
7389        }
7390        let note = match (&summarized, &self.state.change_summary) {
7391            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7392            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7393            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7394            (None, _) => ChangeNote::NotAsked,
7395        };
7396        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7397        if !posting {
7398            let identity = crate::scrub::Identity::current();
7399            return Ok(Some(PrMessage {
7400                title: crate::scrub::scrub(&pr.title, &identity),
7401                body: crate::scrub::scrub(&pr.body, &identity),
7402            }));
7403        }
7404        // This very text was already rejected and asked about: no second
7405        // judge call, rewrite or question, just carry the record on.
7406        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7407        if let Some(known) = self
7408            .state
7409            .github_text
7410            .clone()
7411            .filter(|g| g.fingerprint == raw_fp)
7412        {
7413            return self.settle_withheld(winner, &pr, known).await;
7414        }
7415        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7416        let mut violations =
7417            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7418        let identity = crate::scrub::Identity::current();
7419        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7420            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7421            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7422        {
7423            violations.push(crate::github_text::Violation::SensitiveData);
7424        }
7425        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7426        // only a language violation starts one; sensitive data alone goes
7427        // straight to the owner, with the redacted text as the fallback.
7428        let has_language = violations
7429            .iter()
7430            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7431        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7432            self.state.event(
7433                "github-text",
7434                if has_language {
7435                    format!("description rejected: {violations:?}; requesting one rewrite")
7436                } else {
7437                    format!("description rejected: {violations:?}; asking the owner")
7438                },
7439            );
7440            let mut rewritten = false;
7441            let mut rejected_rewrite = None;
7442            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7443            {
7444                let output = text;
7445                if let Some(title) = summary_title(&output) {
7446                    // Judge the redacted candidate that will actually be posted.
7447                    // Sensitive data alone must not discard a useful rewrite.
7448                    let raw_body = summary_without_title(&output);
7449                    let clean_title = crate::scrub::scrub(&title, &identity);
7450                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7451                    let redacted = clean_title != title || body != raw_body;
7452                    let title = clean_title;
7453                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7454                    if !complete {
7455                        self.state.event(
7456                            "github-text",
7457                            "description rewrite omitted required findings",
7458                        );
7459                    }
7460                    if !body.trim().is_empty() {
7461                        body.push_str("\n\n");
7462                        body.push_str(&public_pr_evidence(&self.state, head));
7463                    }
7464                    let marker = format!("magi:run/{}", self.state.id);
7465                    if !body.trim().is_empty() && !body.contains(&marker) {
7466                        body.push_str(&format!("\n\n{marker}"));
7467                    }
7468                    let rewrite = PrMessage {
7469                        title: title.clone(),
7470                        body: body.clone(),
7471                    };
7472                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7473                    let mut failed = crate::github_text::check_for(
7474                        &self.state.config.graph,
7475                        &title,
7476                        &body,
7477                        redecision,
7478                    );
7479                    if (!crate::github_text::shareable(&title)
7480                        || !crate::github_text::shareable(&body))
7481                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7482                    {
7483                        failed.push(crate::github_text::Violation::SensitiveData);
7484                    }
7485                    if complete
7486                        && !title.trim().is_empty()
7487                        && !body.trim().is_empty()
7488                        && !failed.is_empty()
7489                    {
7490                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7491                            title: title.clone(),
7492                            body: body.clone(),
7493                            failed: failed
7494                                .iter()
7495                                .map(|v| crate::github_text::category(*v).to_owned())
7496                                .collect(),
7497                        });
7498                    }
7499                    if complete
7500                        && !title.trim().is_empty()
7501                        && !raw_body.trim().is_empty()
7502                        && failed.is_empty()
7503                    {
7504                        decision = redecision;
7505                        pr = PrMessage { title, body };
7506                        rewritten = true;
7507                        self.state
7508                            .event("github-text", "description rewrite passed");
7509                        if redacted {
7510                            self.state.event(
7511                                "github-text",
7512                                "sensitive data removed from description rewrite",
7513                            );
7514                        }
7515                    }
7516                }
7517            }
7518            if !rewritten {
7519                self.state.event(
7520                    "github-text",
7521                    "description rewrite unavailable or rejected; asking the owner",
7522                );
7523                if let Some(w) =
7524                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7525                {
7526                    // A different text than the one asked about before: that
7527                    // question no longer describes anything.
7528                    self.retire_github_text_question("the withheld text changed");
7529                    let record = crate::run::GithubTextAsk {
7530                        rewrite: rejected_rewrite,
7531                        fingerprint: raw_fp,
7532                        title: w.title,
7533                        body: w.body,
7534                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7535                        question: None,
7536                        asks: 0,
7537                        resolved: false,
7538                        chosen_title: None,
7539                        choice: None,
7540                    };
7541                    return self.settle_withheld(winner, &pr, record).await;
7542                }
7543            }
7544        }
7545        let (title, body) =
7546            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7547        Ok(Some(PrMessage { title, body }))
7548    }
7549
7550    /// Describe the whole change from its final diff in either run mode.
7551    ///
7552    /// The summary is tied to the final commit and to the merge base of the
7553    /// resolved remote base; a record for any other pair is dropped and the
7554    /// agent asked again, so a description never outlives the diff it was
7555    /// written from. The seat is read-only and the prompt tells it to read the
7556    /// diff and the code around it. `[roles] synthesizer` selects the agents
7557    /// (unset: the default pick), each asked at most once, advancing on an
7558    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7559    /// carries the reason it is unavailable; nothing is invented then.
7560    async fn summarize_change(
7561        &mut self,
7562        winner: &Candidate,
7563        head: Option<&str>,
7564    ) -> std::result::Result<(), String> {
7565        let Some(head) = head else {
7566            return Err("the final commit could not be read".to_owned());
7567        };
7568        let remote = self.state.config.merge.remote.clone();
7569        let tracking = format!("{remote}/{}", self.state.base_branch);
7570        if !git::rev_exists(&winner.worktree, &tracking).await {
7571            return Err(format!("the remote base `{tracking}` could not be read"));
7572        }
7573        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7574            Ok(b) if !b.is_empty() => b,
7575            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7576        };
7577        let evidence = crate::pr_outcome::summary_evidence(
7578            &self.state,
7579            crate::pr_outcome::FinalHead::At(head),
7580        );
7581        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7582        if self
7583            .state
7584            .change_summary
7585            .as_ref()
7586            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7587        {
7588            return Ok(());
7589        }
7590        if self.state.change_summary.take().is_some() {
7591            self.state.event(
7592                "pr-summary",
7593                "the final diff or evidence changed; the earlier change summary was discarded",
7594            );
7595        }
7596        // Save the source snapshot even if no generation agent is available.
7597        if let Err(e) = write_artifact(
7598            &self.state,
7599            &format!("public-description-{fingerprint}.md"),
7600            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7601        ) {
7602            tracing::warn!("could not save public description evidence: {e:#}");
7603        }
7604        let chain = match agent::pick_chain(
7605            &self.state.config.agents,
7606            self.state.config.roles.synthesizer.as_ref(),
7607            &agent::installed,
7608            "synthesizer",
7609        ) {
7610            Ok(chain) => chain,
7611            Err(e) => {
7612                tracing::warn!("no agent for the change summary: {e:#}");
7613                self.state.event("pr-summary", "no summary agent available");
7614                return Err("no agent was available to summarize it".to_owned());
7615            }
7616        };
7617        let subjects = git::subjects(&winner.worktree, &base, head)
7618            .await
7619            .unwrap_or_default();
7620        let stat = git::diff_stat(&winner.worktree, &base, head)
7621            .await
7622            .unwrap_or_default();
7623        let stat: String = stat.chars().take(6_000).collect();
7624        let mut prompt = crate::prompt::change_summary_in(
7625            &base,
7626            head,
7627            &subjects,
7628            stat.trim(),
7629            &self.state.config.graph.github_language,
7630        );
7631        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7632        let sessions = self.state.config.graph.sessions;
7633        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7634        let prompts = self.state.config.prompts.clone();
7635        let cache = self.state.config.cache_dir();
7636        let run = self.state.id.clone();
7637        for (i, spec) in chain.into_iter().enumerate() {
7638            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7639            let job = SeatJob {
7640                spec: spec.clone(),
7641                seat,
7642                cwd: winner.worktree.clone(),
7643                prompt: prompt.clone(),
7644                timeout,
7645                allow_write: false,
7646                sessions,
7647                artifacts: agent::artifacts_dir(&self.state.dir()),
7648                stem: if i == 0 {
7649                    "change-summary".to_owned()
7650                } else {
7651                    format!("change-summary-{}", spec.id)
7652                },
7653                handover: None,
7654            };
7655            let ctx = WaveCtx {
7656                run: &run,
7657                node: "pr-summary",
7658                prompts: &prompts,
7659                cache: cache.as_deref(),
7660                round: None,
7661                carry_seats: false,
7662                pauses: Some((&self.pause, &self.interrupt)),
7663            };
7664            let (seat, outcome) =
7665                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7666            self.state.seats.insert(seat.key.clone(), seat);
7667            let text = match outcome {
7668                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7669                AgentOutcome::ResourceBlocked(why) => {
7670                    // No agent was asked: the run stops resumably before
7671                    // anything is published (see `guarded_pr_message`).
7672                    if self.state.resource_block.is_none() {
7673                        self.state.resource_block = Some(crate::run::ResourceBlock {
7674                            node: "pr-summary".to_owned(),
7675                            kind: "shared resource".to_owned(),
7676                            detail: why,
7677                            at: Timestamp::now(),
7678                        });
7679                    }
7680                    return Err("a shared resource was busy".to_owned());
7681                }
7682                _ => continue,
7683            };
7684            let blind = &self.state.config.blind;
7685            let Some(title) = summary_title(&text) else {
7686                continue;
7687            };
7688            let body: String = summary_without_title(&text)
7689                .lines()
7690                .filter(|l| !l.contains("magi:run/"))
7691                .collect::<Vec<_>>()
7692                .join("\n");
7693            let body = blind::sanitize_prose(body.trim(), blind);
7694            let title = blind::sanitize_prose(&title, blind);
7695            if body.trim().is_empty() || title.trim().is_empty() {
7696                continue;
7697            }
7698            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7699                continue;
7700            }
7701            // The branch must still be the one the agent was shown.
7702            if git::rev_parse(&winner.worktree, "HEAD")
7703                .await
7704                .ok()
7705                .as_deref()
7706                != Some(head)
7707            {
7708                return Err("the branch moved while the summary was written".to_owned());
7709            }
7710            if git::merge_base(&winner.worktree, &tracking, head)
7711                .await
7712                .ok()
7713                .as_deref()
7714                != Some(base.as_str())
7715                || crate::pr_outcome::summary_evidence(
7716                    &self.state,
7717                    crate::pr_outcome::FinalHead::At(head),
7718                ) != evidence
7719            {
7720                return Err("the summary evidence changed while it was written".to_owned());
7721            }
7722            self.state.change_summary = Some(crate::run::ChangeSummary {
7723                head: head.to_owned(),
7724                base,
7725                evidence: fingerprint,
7726                title,
7727                body,
7728            });
7729            self.state.event(
7730                "pr-summary",
7731                format!("change summary written by {} for the final diff", spec.id),
7732            );
7733            if let Err(e) = self.state.save() {
7734                tracing::warn!("could not save the change summary: {e:#}");
7735            }
7736            return Ok(());
7737        }
7738        self.state
7739            .event("pr-summary", "no agent produced a change summary");
7740        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7741    }
7742
7743    /// One read-only rewrite request for a withheld description. The
7744    /// implementer's own conversation is reused only when it exists and can be
7745    /// resumed; otherwise the `[roles] language_judge` chain (or the default
7746    /// pick) writes it in a separate conversation. The prompt carries the whole
7747    /// text, so no agent needs any memory. Each agent is asked at most once and
7748    /// the chain advances on an error, quota, unusable answer or a reply with
7749    /// no `TITLE:` line. Nothing is recorded as a handover.
7750    async fn rewrite_pr_text(
7751        &mut self,
7752        winner: &Candidate,
7753        pr: &PrMessage,
7754        violations: &[crate::github_text::Violation],
7755    ) -> Option<String> {
7756        let prompt = format!(
7757            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7758            pr.title, pr.body
7759        );
7760        let sessions = self.state.config.graph.sessions;
7761        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7762        // (spec, seat, resumed)
7763        let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7764        let key = format!("impl-{}", winner.label);
7765        if let Some(spec) = self
7766            .state
7767            .config
7768            .agents
7769            .iter()
7770            .find(|a| a.id == winner.agent)
7771            .cloned()
7772            && let Some(existing) = self.state.seats.get(&key)
7773            && existing.agent == spec.id
7774            && agent::has_session(spec.kind, existing, sessions)
7775        {
7776            attempts.push((spec, existing.clone(), true));
7777        } else {
7778            let chain = agent::pick_chain(
7779                &self.state.config.agents,
7780                self.state.config.roles.language_judge.as_ref(),
7781                &agent::installed,
7782                "language_judge",
7783            );
7784            match chain {
7785                Ok(chain) => {
7786                    for spec in chain {
7787                        let seat = crate::fixer::seat_for(
7788                            &mut self.state,
7789                            "github-text-rewrite",
7790                            &spec.id,
7791                        );
7792                        attempts.push((spec, seat, false));
7793                    }
7794                }
7795                Err(e) => {
7796                    tracing::warn!("no agent for the description rewrite: {e:#}");
7797                    self.state
7798                        .event("github-text", "no rewrite agent available");
7799                    return None;
7800                }
7801            }
7802        }
7803        let prompts = self.state.config.prompts.clone();
7804        let cache = self.state.config.cache_dir();
7805        let run = self.state.id.clone();
7806        for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7807            let job = SeatJob {
7808                spec: spec.clone(),
7809                seat,
7810                cwd: winner.worktree.clone(),
7811                prompt: prompt.clone(),
7812                timeout: retry_budget(timeout_full, resumed),
7813                allow_write: false,
7814                sessions,
7815                artifacts: agent::artifacts_dir(&self.state.dir()),
7816                stem: if i == 0 {
7817                    "github-text-rewrite".to_owned()
7818                } else {
7819                    format!("github-text-rewrite-{}", spec.id)
7820                },
7821                handover: None,
7822            };
7823            let ctx = WaveCtx {
7824                run: &run,
7825                node: "github-text",
7826                prompts: &prompts,
7827                cache: cache.as_deref(),
7828                round: None,
7829                carry_seats: false,
7830                pauses: Some((&self.pause, &self.interrupt)),
7831            };
7832            let (seat, outcome) =
7833                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7834            self.state.seats.insert(seat.key.clone(), seat);
7835            match outcome {
7836                AgentOutcome::Ok(o)
7837                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7838                {
7839                    return Some(o.text);
7840                }
7841                AgentOutcome::ResourceBlocked(_) => return None,
7842                _ => {}
7843            }
7844        }
7845        None
7846    }
7847
7848    /// Abandon the standing posting-gate question, if any.
7849    fn retire_github_text_question(&mut self, why: &str) {
7850        let Some(id) = self
7851            .state
7852            .github_text
7853            .as_ref()
7854            .and_then(|g| g.question.clone())
7855        else {
7856            return;
7857        };
7858        let _ = ask::Questions::open().update(&id, |q| {
7859            q.abandon(why);
7860            Ok(())
7861        });
7862    }
7863
7864    /// Carry a withheld text to its end: file the question, park while it is
7865    /// open, and on an answer, silence or timeout pick the text to post. The
7866    /// decision is saved before it is used, so it applies once.
7867    async fn settle_withheld(
7868        &mut self,
7869        winner: &Candidate,
7870        pr: &PrMessage,
7871        mut g: crate::run::GithubTextAsk,
7872    ) -> Result<Option<PrMessage>> {
7873        use crate::github_text as gt;
7874        let store = ask::Questions::open();
7875        let timeout = self.state.config.graph.answer_timeout;
7876        if !g.resolved {
7877            let mut standing = g
7878                .question
7879                .as_ref()
7880                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7881            if let Some(q) = standing.as_ref()
7882                && q.status.open()
7883                && gt::expired(q, timeout)
7884            {
7885                let why = format!("no answer within {}s of asking", timeout.max(1));
7886                standing = store
7887                    .update(&q.id, |q| {
7888                        q.abandon(&why);
7889                        Ok(())
7890                    })
7891                    .ok()
7892                    .map(|(q, ())| q);
7893            }
7894            let mut ask_again: Option<Option<Vec<String>>> = None;
7895            match standing {
7896                // Recorded but never written (a stop between the two saves),
7897                // or removed: ask again without spending the retry.
7898                None if g.asks <= 1 => {
7899                    g.asks = 0;
7900                    ask_again = Some(None);
7901                }
7902                None => {
7903                    g.resolved = true;
7904                    g.chosen_title = None;
7905                    g.choice = Some(GithubTextChoice::Fallback);
7906                }
7907                Some(q) if q.status.open() => {
7908                    self.park_on_github_text(&g)?;
7909                    return Ok(None);
7910                }
7911                Some(q) => match gt::read_reply(&q) {
7912                    gt::Reply::Fallback => {
7913                        g.resolved = true;
7914                        g.chosen_title = None;
7915                        g.choice = Some(GithubTextChoice::Fallback);
7916                    }
7917                    gt::Reply::Original => {
7918                        g.resolved = true;
7919                        g.chosen_title = None;
7920                        g.choice = Some(GithubTextChoice::Original);
7921                    }
7922                    gt::Reply::Rewrite => {
7923                        g.resolved = true;
7924                        g.chosen_title = None;
7925                        g.choice = Some(if g.rewrite.is_some() {
7926                            GithubTextChoice::Rewrite
7927                        } else {
7928                            GithubTextChoice::Fallback
7929                        });
7930                    }
7931                    gt::Reply::Title(text) => {
7932                        let first = text
7933                            .lines()
7934                            .map(str::trim)
7935                            .find(|l| !l.is_empty())
7936                            .unwrap_or("");
7937                        let decision = if first.is_empty() {
7938                            None
7939                        } else {
7940                            let candidate = PrMessage {
7941                                title: first.to_owned(),
7942                                body: String::new(),
7943                            };
7944                            self.judge_pr_language(&winner.worktree, &candidate).await
7945                        };
7946                        match gt::vet_title(
7947                            &text,
7948                            decision,
7949                            self.state.config.graph.github_english(),
7950                        ) {
7951                            Ok(title) => {
7952                                g.resolved = true;
7953                                g.chosen_title = Some(title);
7954                                g.choice = Some(GithubTextChoice::MyTitle);
7955                            }
7956                            Err(cats) if g.asks < 2 => {
7957                                ask_again = Some(Some(
7958                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7959                                ))
7960                            }
7961                            Err(_) => {
7962                                g.resolved = true;
7963                                g.chosen_title = None;
7964                                g.choice = Some(GithubTextChoice::Fallback);
7965                            }
7966                        }
7967                    }
7968                },
7969            }
7970            if let Some(rejected) = ask_again {
7971                // A retry (after a rejected replacement title) is still about
7972                // the original text: the same ORIGINAL / FALLBACK comparison.
7973                let w = gt::Withheld {
7974                    title: g.title,
7975                    body: g.body,
7976                    categories: g.categories.clone(),
7977                };
7978                // Named by the original text's fingerprint alone, so a resume
7979                // rewrites the same file and the detail stays identical.
7980                let artifact = match crate::run::write_artifact(
7981                    &self.state,
7982                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7983                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7984                ) {
7985                    Ok(p) => Some(p),
7986                    Err(e) => {
7987                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7988                        None
7989                    }
7990                };
7991                let (detail, comparison) = gt::question_content(
7992                    &w,
7993                    &pr.title,
7994                    &pr.body,
7995                    &self.state.id,
7996                    rejected.as_deref(),
7997                    artifact.as_deref(),
7998                    g.rewrite.as_ref(),
7999                );
8000                let mut q = ask::Question::new(
8001                    self.state.id.clone(),
8002                    gt::ASK_NODE.to_owned(),
8003                    gt::ASK_SEAT.to_owned(),
8004                    gt::question_summary(&self.state.config.graph.language, &w),
8005                    detail,
8006                    gt::question_choices(&w, g.rewrite.as_ref()),
8007                );
8008                q.posting_comparison = Some(comparison);
8009                q.answer_timeout = timeout;
8010                // Recorded before it is written: a stop in between leaves a
8011                // record naming a question the resume then files again.
8012                g.question = Some(q.id.clone());
8013                g.asks += 1;
8014                self.state.github_text = Some(g.clone());
8015                self.state.event(
8016                    "github-text",
8017                    format!("asking the owner about the withheld text ({})", q.short()),
8018                );
8019                self.state.save()?;
8020                store
8021                    .put_executable(&mut q)
8022                    .context("file the posting-gate question")?;
8023                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8024                    tracing::warn!(
8025                        "could not notify about posting-gate question {}: {e:#}",
8026                        q.short()
8027                    );
8028                }
8029                self.park_on_github_text(&g)?;
8030                return Ok(None);
8031            }
8032        }
8033        let resolved = g.chosen_title.clone();
8034        let original = g.choice == Some(GithubTextChoice::Original);
8035        // Field names only: the event log never records the text.
8036        self.state.event(
8037            "github-text",
8038            if original {
8039                format!(
8040                    "posting the original text for: {}",
8041                    match (g.title, g.body) {
8042                        (true, true) => "title, description",
8043                        (true, false) => "title",
8044                        _ => "description",
8045                    }
8046                )
8047            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8048                "posting the saved rewrite title and description".to_owned()
8049            } else if resolved.is_some() {
8050                "posting the owner's replacement title".to_owned()
8051            } else {
8052                "posting the neutral text for the withheld fields".to_owned()
8053            },
8054        );
8055        self.state.github_text = Some(g.clone());
8056        self.state.save()?;
8057        if g.choice == Some(GithubTextChoice::Rewrite)
8058            && let Some(r) = g.rewrite.as_ref()
8059        {
8060            return Ok(Some(PrMessage {
8061                title: r.title.clone(),
8062                body: r.body.clone(),
8063            }));
8064        }
8065        let w = gt::Withheld {
8066            title: g.title,
8067            body: g.body,
8068            categories: g.categories.clone(),
8069        };
8070        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8071        if let Some(t) = resolved {
8072            title = t;
8073        }
8074        // Every withheld field is now neutral or vetted English.
8075        let english = gt::LanguageDecision {
8076            title_english: true,
8077            body_english: true,
8078            body_complete: true,
8079        };
8080        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8081        if original {
8082            // The owner's explicit choice overrides the gate for the fields it
8083            // was made for; the others keep what the gate produced.
8084            if g.title {
8085                title = pr.title.clone();
8086            }
8087            if g.body {
8088                body = pr.body.clone();
8089            }
8090        }
8091        Ok(Some(PrMessage { title, body }))
8092    }
8093
8094    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8095        self.state.github_text = Some(g.clone());
8096        self.state.end_pass_parked(ParkCause::Question);
8097        self.state.parked = true;
8098        self.state.event(
8099            "github-text",
8100            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8101        );
8102        self.state.save()
8103    }
8104
8105    /// Ask `[roles] language_judge` about this text and record which source
8106    /// decided; the text itself is never recorded.
8107    async fn judge_pr_language(
8108        &mut self,
8109        cwd: &Path,
8110        pr: &PrMessage,
8111    ) -> Option<crate::github_text::LanguageDecision> {
8112        self.state.config.roles.language_judge.as_ref()?;
8113        if !self.state.config.graph.language_gate() {
8114            return None;
8115        }
8116        let decision =
8117            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8118        self.state.event(
8119            "github-text",
8120            if decision.is_some() {
8121                "language decided by the language judge"
8122            } else {
8123                "language judge unavailable; using built-in heuristics"
8124            },
8125        );
8126        decision
8127    }
8128
8129    async fn merge(&mut self) -> Result<()> {
8130        // Same reasoning as `gate`: ask the review and gate records directly
8131        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8132        // skip can rewrite on reentry to something that no longer says
8133        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8134        // so a hand-off (open findings, green verification) reaches merge
8135        // exactly like a genuinely clean round does.
8136        //
8137        // A run resumed mid-`land` never reaches here at all: `execute`
8138        // recognises `RunStatus::Landing` before it even calls `prep`, and
8139        // routes straight to `run_land` instead. That has to happen a level
8140        // up from this function, not with a check in here, because
8141        // `review_loop`'s own status recomputation (see its doc) runs
8142        // *before* `merge` on every reentry and would otherwise overwrite
8143        // the `Landing` marker with `Gating` before this node ever saw it.
8144        if self
8145            .state
8146            .base_sync
8147            .as_ref()
8148            .is_some_and(|s| s.conflict.is_some())
8149            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8150                != Some(RunStatus::Gating)
8151            // `gate_ran == false` is not "passed" - `gate` leaves it false
8152            // both before it has ever run and when its last attempt was
8153            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8154            // permission to merge on nothing but the review record. Only a
8155            // gate that actually ran may proceed; an intentional local
8156            // review can also finish with verification explicitly not configured.
8157            // `RunState::gate_status` computes that distinction.
8158            || !self.state.gate_allows_progress()
8159        {
8160            return Ok(());
8161        }
8162        // This node's own record, not `status`: `status == Ready` is not
8163        // unique to the harmless `MergeMode::None` path this line was
8164        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8165        // run's PR was closed without merging — and on that run `mode` is
8166        // still `Pr`, so a reentry that fell through here would push and
8167        // open a second pull request. `self.state.merge` is set exactly once
8168        // this node (or `land`) has already produced a verdict, under every
8169        // mode, which is what "already done" actually means here.
8170        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8171            return Ok(());
8172        }
8173        let Some(winner) = self.state.winner().cloned() else {
8174            return Ok(());
8175        };
8176        let repo = self.state.repo.clone();
8177        let base = self.state.base_branch.clone();
8178        let mode = self.state.config.merge.mode;
8179        let style = self.state.config.merge.style;
8180        let facts = if is_review_run(&self.state) {
8181            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8182            let start = review_base(
8183                &repo,
8184                &self.state.config.merge.remote,
8185                &base,
8186                &self.state.base_commit,
8187                &winner.branch,
8188            )
8189            .await;
8190            branch_facts(&repo, &start, &winner.branch).await
8191        } else {
8192            None
8193        };
8194        // `None` when the base could not be freshly read: then an adopted
8195        // pull request's title is left alone.
8196        let leaked = if is_review_run(&self.state) {
8197            leaked_subjects(&self.state, &winner.branch).await
8198        } else {
8199            Some(Vec::new())
8200        };
8201        // `None`: parked on the owner's word about a withheld title or body.
8202        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8203        // the resume arrives here again.
8204        let Some(pr) = self
8205            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8206            .await?
8207        else {
8208            return Ok(());
8209        };
8210        let message = pr.commit_message();
8211
8212        let outcome = match mode {
8213            MergeMode::None => MergeOutcome {
8214                mode,
8215                ok: true,
8216                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8217                empty: false,
8218            },
8219            MergeMode::Pr | MergeMode::Local
8220                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8221            {
8222                MergeOutcome {
8223                    mode,
8224                    ok: false,
8225                    detail: empty_candidate_detail(&self.state, &base),
8226                    empty: true,
8227                }
8228            }
8229            MergeMode::Local => {
8230                let on = git::current_branch(&repo).await?;
8231                if on.as_deref() != Some(base.as_str()) {
8232                    MergeOutcome {
8233                        mode,
8234                        ok: false,
8235                        detail: format!(
8236                            "{} has {} checked out, not the base branch {base}",
8237                            repo.display(),
8238                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8239                        ),
8240                        empty: false,
8241                    }
8242                } else if !git::is_clean(&repo).await? {
8243                    MergeOutcome {
8244                        mode,
8245                        ok: false,
8246                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8247                        empty: false,
8248                    }
8249                } else {
8250                    let out = match style {
8251                        MergeStyle::Merge => {
8252                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8253                        }
8254                        MergeStyle::Squash => {
8255                            git::merge_squash(&repo, &winner.branch, &message).await?
8256                        }
8257                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8258                    };
8259                    MergeOutcome {
8260                        mode,
8261                        ok: out.ok(),
8262                        detail: if out.ok() { out.stdout } else { out.stderr },
8263                        empty: false,
8264                    }
8265                }
8266            }
8267            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8268                self.state.pr_publication_pending = true;
8269                self.state.save()?;
8270                let out =
8271                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8272                self.state.pr_publication_pending = out.is_err();
8273                MergeOutcome {
8274                    mode,
8275                    ok: out.is_ok(),
8276                    detail: match out {
8277                        Ok(url) => url,
8278                        Err(e) => format!("{e:#}"),
8279                    },
8280                    empty: false,
8281                }
8282            }
8283            MergeMode::Pr => {
8284                let remote = self.state.config.merge.remote.clone();
8285                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8286                if !pushed.ok() {
8287                    MergeOutcome {
8288                        mode,
8289                        ok: false,
8290                        detail: pushed.stderr,
8291                        empty: false,
8292                    }
8293                } else {
8294                    // A retry or resume of a run whose branch already has an
8295                    // open pull request adopts it rather than failing on a
8296                    // duplicate. Only this winner branch into this base:
8297                    // `branch_for` derives the name from the run id, so a
8298                    // different run's pull request never matches.
8299                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8300                    let out = match pr_merge_plan(found) {
8301                        PrPlan::Create => {
8302                            gh_pr_create(
8303                                &winner.worktree,
8304                                &base,
8305                                &winner.branch,
8306                                &pr.title,
8307                                &pr.body,
8308                            )
8309                            .await
8310                        }
8311                        PrPlan::Adopt { url, title } => {
8312                            self.state
8313                                .event("merge", format!("Pr: adopted open pull request {url}"));
8314                            if self.state.github_text.as_ref().is_some_and(|g| {
8315                                g.resolved
8316                                    && g.choice == Some(GithubTextChoice::Rewrite)
8317                                    && g.rewrite.is_some()
8318                            }) {
8319                                // The owner chose the entire pair, even if the
8320                                // existing title already matches or was edited.
8321                                land::set_pr_text_verbatim(
8322                                    &winner.worktree,
8323                                    &url,
8324                                    &pr.title,
8325                                    &pr.body,
8326                                )
8327                                .await
8328                                .map(|()| url)
8329                            } else {
8330                                if title != pr.title
8331                                    && (!is_review_run(&self.state)
8332                                        || leaked
8333                                            .as_deref()
8334                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8335                                    && let Err(e) =
8336                                        if self.state.github_text.as_ref().is_some_and(|g| {
8337                                            g.resolved
8338                                                && g.title
8339                                                && g.choice == Some(GithubTextChoice::Original)
8340                                        }) {
8341                                            // The owner chose to post this title unmodified.
8342                                            land::set_pr_title_verbatim(
8343                                                &winner.worktree,
8344                                                &url,
8345                                                &pr.title,
8346                                            )
8347                                            .await
8348                                        } else {
8349                                            land::set_pr_title(
8350                                                &mut self.state,
8351                                                &winner.worktree,
8352                                                &url,
8353                                                &pr.title,
8354                                            )
8355                                            .await
8356                                        }
8357                                {
8358                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8359                                    self.state
8360                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8361                                }
8362                                Ok(url)
8363                            }
8364                        }
8365                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8366                    };
8367                    match out {
8368                        Ok(url) => MergeOutcome {
8369                            mode,
8370                            ok: true,
8371                            detail: url,
8372                            empty: false,
8373                        },
8374                        Err(e) => MergeOutcome {
8375                            mode,
8376                            ok: false,
8377                            detail: e.to_string(),
8378                            empty: false,
8379                        },
8380                    }
8381                }
8382            }
8383        };
8384
8385        self.state.status = match (mode, outcome.ok) {
8386            (MergeMode::None, _) => RunStatus::Ready,
8387            (_, true) if outcome.empty => RunStatus::Ready,
8388            (MergeMode::Pr, true) => RunStatus::Ready,
8389            (_, true) => RunStatus::Merged,
8390            (_, false) => RunStatus::Blocked,
8391        };
8392        self.state.event(
8393            "merge",
8394            format!(
8395                "{:?}: {}",
8396                mode,
8397                outcome.detail.lines().next().unwrap_or("")
8398            ),
8399        );
8400        self.state.merge = Some(outcome);
8401        self.state.save()?;
8402
8403        // The PR is open and the run would historically stop here, leaving the
8404        // operator to watch checks, feed review comments back to a fixer, and
8405        // merge. That was done by hand six times in one session before this
8406        // existed. Opt-in, because merging is the one irreversible thing magi
8407        // can do to a repository.
8408        if self.state.config.graph.land
8409            && mode == MergeMode::Pr
8410            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8411        {
8412            self.run_land().await?;
8413        }
8414        // `run_land` may have left `status` at `Landing` - still waiting on
8415        // CI or the owner's approval, not actually settled - so this has to
8416        // read whatever `status` ended up as here, not the `Merged` this
8417        // function set a few lines up.
8418        self.settle_questions();
8419        Ok(())
8420    }
8421
8422    /// Enter `land`.
8423    ///
8424    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8425    /// a resumed run's re-entry. `land::land` itself is what serialises the
8426    /// two git-mutating moments inside the loop — the rebase push and
8427    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8428    /// to hold a lock across the whole call, and doing so would serialise
8429    /// this run's CI wait against a *different* run's land-approval resume
8430    /// in the same repository, which is exactly the "must not wait on
8431    /// another task" property the daemon's slot-freeing exists to give.
8432    async fn run_land(&mut self) -> Result<()> {
8433        crate::pr_head::ensure_owner(&self.state)?;
8434        let url = self
8435            .state
8436            .merge
8437            .as_ref()
8438            .map(|m| m.detail.clone())
8439            .unwrap_or_default();
8440        let url = url.lines().next().unwrap_or("").trim().to_owned();
8441        if !url.starts_with("http") {
8442            return Ok(());
8443        }
8444        // A land failure is not a lost run: the work is on a branch and the
8445        // pull request is open, which is exactly where a human takes over.
8446        loop {
8447            match land::land(&mut self.state, &url).await {
8448                Ok(_)
8449                    if self
8450                        .state
8451                        .merge_repairs
8452                        .last()
8453                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8454                {
8455                    if self.repair_merge_review().await? {
8456                        continue;
8457                    }
8458                }
8459                Ok(pr) if self.state.parked => {
8460                    // `land` already saved the parked marker; nothing here
8461                    // overrides `status` back to a terminal value while an
8462                    // approval is still outstanding.
8463                    let _ = pr;
8464                }
8465                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8466                    self.state.status = RunStatus::Superseded;
8467                    self.state.save()?;
8468                }
8469                Ok(pr) => {
8470                    self.state.status = match pr.state {
8471                        land::PrLifecycle::Merged => RunStatus::Merged,
8472                        _ => RunStatus::Blocked,
8473                    };
8474                    // Downstream of a confirmed merge only - see
8475                    // `bump::should_release_bump`'s own doc for why this one
8476                    // check covers all three of `land`'s success paths.
8477                    // Best-effort: the run already landed, so a failure here
8478                    // (the decision call, `gh`, `cargo`) is recorded and never
8479                    // turns a landed run into a failed one.
8480                    if bump::should_release_bump(self.state.status)
8481                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8482                    {
8483                        // The event is the run's own record. Not-eligible cases
8484                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8485                        // `Err` is a bump that was tried and failed:
8486                        // `after_merge` itself raises the operator notice for
8487                        // that, whether or not a release PR exists yet.
8488                        self.state
8489                            .event("bump", format!("release bump skipped: {e:#}"));
8490                    }
8491                    // Independent of the bump, and best-effort in the same way:
8492                    // findings the merge left open become follow-up tasks.
8493                    if self.state.status == RunStatus::Merged {
8494                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8495                    }
8496                    self.state.save()?;
8497                }
8498                Err(e) => {
8499                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8500                        return Err(e);
8501                    }
8502                    self.state.status = RunStatus::Blocked;
8503                    self.state.event("land", format!("gave up: {e}"));
8504                    self.state.save()?;
8505                }
8506            }
8507            break;
8508        }
8509        Ok(())
8510    }
8511
8512    /// One repair cycle stays under the existing driver/task claim. An
8513    /// interrupted fixer is rolled back and held rather than run twice.
8514    async fn repair_merge_review(&mut self) -> Result<bool> {
8515        use crate::run::MergeRepairPhase as Phase;
8516        crate::pr_head::ensure_owner(&self.state)?;
8517        let i = self.state.merge_repairs.len() - 1;
8518        let request = self.state.merge_repairs[i].clone();
8519        if request.phase == Phase::Fixing {
8520            let restored = match self.state.winner().cloned() {
8521                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8522                None => Err(anyhow::anyhow!("repair has no winning branch")),
8523            };
8524            let why = match restored {
8525                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8526                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8527                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8528            };
8529            self.hold_merge_repair(i, why)?;
8530            return Ok(false);
8531        }
8532        let failure = if i >= self.state.config.graph.land_rounds
8533            || self.state.config.graph.review_rounds == 0
8534        {
8535            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8536        } else {
8537            None
8538        };
8539        if let Some(why) = failure {
8540            self.hold_merge_repair(i, why)?;
8541            return Ok(false);
8542        }
8543        if request.phase == Phase::Requested {
8544            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8545                Ok(pr) => pr,
8546                Err(e) => {
8547                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8548                    return Ok(false);
8549                }
8550            };
8551            self.state.merge_repairs[i].phase = Phase::Fixing;
8552            self.state.land_approval = None;
8553            self.state.land_armed_head = None;
8554            self.state.save()?;
8555            // Serialize the exact records, including finding ids/details,
8556            // reconsideration reasons, final votes and verification output.
8557            let evidence = serde_json::to_string_pretty(&request)?;
8558            let reason = format!(
8559                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8560            );
8561            let budget = self.state.config.graph.land_rounds;
8562            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8563            match result {
8564                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8565                Ok(land::Fixed::Failed(why)) => {
8566                    self.hold_merge_repair(
8567                        i,
8568                        format!(
8569                            "repair failed: {why}; inspect the branch and run an explicit review"
8570                        ),
8571                    )?;
8572                    return Ok(false);
8573                }
8574                Err(e) => {
8575                    self.hold_merge_repair(
8576                        i,
8577                        format!(
8578                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8579                        ),
8580                    )?;
8581                    return Ok(false);
8582                }
8583            }
8584            // Even no change gets a real new reviewer wave. The previous
8585            // evidence remains in the request; a former clean/concluded round
8586            // cannot short-circuit the new review or gate.
8587            self.begin_merge_repair_review(i)?;
8588        }
8589        self.review_loop().await?;
8590        self.gate().await?;
8591        if self.resource_halt()? || self.state.parked {
8592            return Ok(false);
8593        }
8594        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8595            != Some(RunStatus::Gating)
8596            || !self.state.gate_allows_progress()
8597        {
8598            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8599            return Ok(false);
8600        }
8601        // Review and gate may have added commits after the initial repair.
8602        // Publish exactly their final head to the existing PR, with a lease.
8603        let winner = self
8604            .state
8605            .winner()
8606            .cloned()
8607            .context("repair has no winning branch")?;
8608        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8609        if self.state.gate_head.as_deref() != Some(after.as_str())
8610            || !git::is_clean(&winner.worktree).await?
8611        {
8612            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8613            return Ok(false);
8614        }
8615        let expected = self
8616            .state
8617            .pr_pushed_head
8618            .as_deref()
8619            .unwrap_or(&request.head)
8620            .to_owned();
8621        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8622        // Push the checked commit itself: a local ref moving after the check
8623        // must not change which commit the forge receives.
8624        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8625        let refspec = format!("{after}:refs/heads/{public}");
8626        let push = git::git_raw(
8627            &winner.worktree,
8628            &["push", &lease, &self.state.config.merge.remote, &refspec],
8629        )
8630        .await;
8631        match push {
8632            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8633            Ok(out) => {
8634                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8635                return Ok(false);
8636            }
8637            Err(e) => {
8638                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8639                return Ok(false);
8640            }
8641        }
8642        self.state.merge_repairs[i].phase = Phase::Complete;
8643        self.state.status = RunStatus::Landing;
8644        self.state.event(
8645            "repair",
8646            "new review and gate completed; fresh owner merge decision required",
8647        );
8648        self.state.save()?;
8649        Ok(true)
8650    }
8651
8652    fn merge_repair_artifacts(&self) -> PathBuf {
8653        let artifacts = agent::artifacts_dir(&self.state.dir());
8654        match self
8655            .state
8656            .merge_repairs
8657            .last()
8658            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8659        {
8660            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8661            None => artifacts,
8662        }
8663    }
8664
8665    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8666        self.state.reviews.clear();
8667        self.state.gate.clear();
8668        self.state.gate_ran = false;
8669        self.state.gate_head = None;
8670        self.state.gate_fixes.clear();
8671        self.state.pre_gate.clear();
8672        self.state.pre_gate_commit = None;
8673        self.state.contested_handoff = None;
8674        self.state.land_reruns = None;
8675        self.state.land_approval = None;
8676        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8677        self.state.status = RunStatus::Reviewing;
8678        self.state.save()
8679    }
8680
8681    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8682        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8683        self.state.merge_repairs[index].failure = Some(why.clone());
8684        self.state.status = RunStatus::Blocked;
8685        self.state.event("repair", why);
8686        self.state.save()
8687    }
8688
8689    // -------------------------------------------------------------- helpers
8690
8691    /// Fetch or create a seat, keeping its conversation across nodes.
8692    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8693        if let Some(existing) = self.state.seats.get(key)
8694            && existing.agent == agent
8695        {
8696            return existing.clone();
8697        }
8698        // A seat that changes agent mints its session id from the agent too,
8699        // like a handover: the old agent's uuid is already taken by the CLI.
8700        let fresh = if self.state.seats.contains_key(key) {
8701            handover_seat(key, agent, self.state.next_seat_seed())
8702        } else {
8703            SeatState::new(key, agent, self.state.seed)
8704        };
8705        self.state.seats.insert(key.to_owned(), fresh.clone());
8706        fresh
8707    }
8708
8709    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8710    /// seat to another roster agent, in which case that agent. Nodes that
8711    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8712    /// reconsideration) must keep talking to whoever answered it, not slip
8713    /// back to the agent that failed it.
8714    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8715        match self.state.seats.get(key) {
8716            Some(s) if s.agent != spec.id => {
8717                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8718            }
8719            _ => spec,
8720        }
8721    }
8722
8723    /// A candidate rendered for judging, with the leak policy applied.
8724    fn view(&self, c: &Candidate) -> CandidateView {
8725        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8726            .unwrap_or_default();
8727        let (patch, _) = blind::sanitize_patch(
8728            &format!("candidate {} patch", c.label),
8729            &raw,
8730            &self.state.config.blind,
8731        );
8732        CandidateView {
8733            label: c.label,
8734            branch: c.branch.clone(),
8735            summary: c.summary.clone(),
8736            stat: c.stat.clone(),
8737            patch,
8738        }
8739    }
8740
8741    /// The full candidate set as prompt text, for seats with no live session.
8742    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8743        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8744        prompt::judge(
8745            "(see above)",
8746            &views,
8747            self.roles.judges.len(),
8748            base_short,
8749            "en",
8750        )
8751    }
8752
8753    /// The final-vote prompt with everything a seat that has no session of its
8754    /// own needs: the candidates, the seat's own ranking and reasons, and the
8755    /// anonymised deliberation it took part in (only when there was one, so a
8756    /// handed-over seat never sees more than the seat it replaces did). The
8757    /// `Final vote` heading stays first.
8758    fn vote_prompt_full(
8759        &self,
8760        j: usize,
8761        labels: &[char],
8762        language: &str,
8763        candidates: &[Candidate],
8764        base_short: &str,
8765    ) -> String {
8766        let mut text = format!(
8767            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8768            prompt::final_vote(labels, language),
8769            self.state.instruction,
8770            self.candidate_block(candidates, base_short)
8771        );
8772        if let Some(own) = self
8773            .state
8774            .judgements
8775            .get(j)
8776            .filter(|r| !r.ranking.is_empty())
8777        {
8778            let reasons = own
8779                .reasons
8780                .iter()
8781                .map(|(k, v)| format!("- {k}: {v}"))
8782                .collect::<Vec<_>>()
8783                .join("\n");
8784            text.push_str(&format!(
8785                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8786                own.ranking.iter().collect::<String>(),
8787                if reasons.is_empty() {
8788                    ""
8789                } else {
8790                    ", because:\n"
8791                }
8792            ));
8793        }
8794        if !self.state.deliberation.is_empty() {
8795            text.push_str("\n# What was argued before this vote\n");
8796            for t in self.transcript(&[], j) {
8797                text.push_str(&format!(
8798                    "\n## {}{}\n\n{}\n",
8799                    t.who,
8800                    if t.is_self { " (you)" } else { "" },
8801                    t.body.trim()
8802                ));
8803            }
8804        }
8805        text
8806    }
8807
8808    /// Anonymised transcript for judge `self_idx`.
8809    ///
8810    /// The initial rankings are always the opening statements. Seeding them
8811    /// only when no turn had been taken yet meant every judge after the first
8812    /// argued against a single voice instead of against the actual split — the
8813    /// disagreement is the information, so it is always on the table.
8814    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8815        let mut turns = Vec::new();
8816        for j in &self.state.judgements {
8817            if j.ranking.is_empty() {
8818                continue;
8819            }
8820            let reasons = j
8821                .reasons
8822                .iter()
8823                .map(|(k, v)| format!("- {k}: {v}"))
8824                .collect::<Vec<_>>()
8825                .join("\n");
8826            turns.push(Turn {
8827                who: format!("Judge {} (opening ranking)", j.judge),
8828                is_self: j.judge == self_idx + 1,
8829                body: format!(
8830                    "Ranked {}{}{reasons}",
8831                    j.ranking.iter().collect::<String>(),
8832                    if reasons.is_empty() {
8833                        ""
8834                    } else {
8835                        ", because:\n"
8836                    }
8837                ),
8838            });
8839        }
8840        for t in self
8841            .state
8842            .deliberation
8843            .iter()
8844            .flat_map(|r| r.turns.iter())
8845            .chain(current)
8846        {
8847            turns.push(Turn {
8848                who: format!("Judge {}", t.judge),
8849                is_self: t.judge == self_idx + 1,
8850                body: t.body.clone(),
8851            });
8852        }
8853        turns
8854    }
8855}
8856
8857/// Does this seat still hold the context a follow-up prompt would rely on?
8858fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8859    agent::has_session(spec.kind, seat, sessions)
8860}
8861
8862/// The next entry in `roster` after `start`, never wrapping back to the
8863/// front, whose id is not in `tried` yet.
8864///
8865/// Starts one past `start` rather than at the front of `roster`: `start` is
8866/// the seat's own original position, and a seat whose candidate slot already
8867/// sits on the roster's second entry must fall through to the third next, not
8868/// restart at the first — which is very likely a different candidate's own
8869/// agent already. Never wraps back past `start`, for the same reason: an
8870/// entry earlier in the roster than the seat's own position is almost
8871/// certainly some *other* candidate slot's own agent, and once the tail of
8872/// the roster is exhausted there are no more untried agents for *this* seat
8873/// to fall through to — the caller's fallback chain ends there, exactly as
8874/// "no further untried agents remain in the list for that seat" asks for.
8875///
8876/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8877/// the same id twice (an operator's `roles.implementers` typo, or a
8878/// `[[agents]]` list reused across roles) must not let
8879/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8880/// over `roster` either finds an untried id or runs out, so this always
8881/// terminates regardless of duplicates.
8882fn next_untried_in_roster<'a>(
8883    roster: &'a [AgentSpec],
8884    start: usize,
8885    tried: &BTreeSet<String>,
8886) -> Option<&'a AgentSpec> {
8887    roster
8888        .get(start + 1..)?
8889        .iter()
8890        .find(|s| !tried.contains(&s.id))
8891}
8892
8893/// The successor for a seat, shared by all four seat kinds (implement, judge,
8894/// review, advise). `others` holds the ids that *currently* occupy the other
8895/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8896/// sees them), so roster entries beyond the seat count act as spares: a failed
8897/// seat goes to an agent no other seat holds whenever the roster permits.
8898///
8899/// `carried` is `Some` only for the review loop's carried failure history.
8900/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8901/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8902/// whole roster: untried and not another seat's occupant; (3) the plain walk
8903/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8904/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8905/// a better outcome than an empty seat, and it keeps the answer to "is there
8906/// a successor at all" exactly what it was before occupants were considered,
8907/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8908/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8909/// carried failure is a cheaper bet than doubling an agent on the panel, and
8910/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8911/// the same round are handled one at a time, so a seat later in the batch
8912/// sees an earlier one's new occupant but not yet its own freed agent.
8913fn pick_successor<'a>(
8914    roster: &'a [AgentSpec],
8915    start: usize,
8916    tried: &BTreeSet<String>,
8917    carried: Option<&BTreeSet<String>>,
8918    others: &BTreeSet<String>,
8919) -> Option<&'a AgentSpec> {
8920    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8921    roster
8922        .get(start + 1..)
8923        .and_then(|tail| {
8924            tail.iter()
8925                .filter(free)
8926                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8927        })
8928        .or_else(|| {
8929            carried
8930                .is_some()
8931                .then(|| roster.iter().find(free))
8932                .flatten()
8933        })
8934        .or_else(|| match carried {
8935            Some(c) => next_for_seat(roster, start, tried, c),
8936            None => next_untried_in_roster(roster, start, tried),
8937        })
8938}
8939
8940/// The next agent for a seat that carries its failure history across rounds
8941/// (the review loop). `round_tried` is this round's own bound and starts
8942/// empty every round; `carried_failed` only decides priority.
8943///
8944/// First: an id walking forward from `start`, never wrapping, that is neither
8945/// tried this round nor failed in an earlier one. Only when that is exhausted
8946/// does it rescue: the first roster id (in roster order, so this one *does*
8947/// look before `start`) not yet tried this round, which is by then a carried
8948/// failure. Each id is rescued at most once per round, so it cannot loop.
8949fn next_for_seat<'a>(
8950    roster: &'a [AgentSpec],
8951    start: usize,
8952    round_tried: &BTreeSet<String>,
8953    carried_failed: &BTreeSet<String>,
8954) -> Option<&'a AgentSpec> {
8955    roster
8956        .get(start + 1..)?
8957        .iter()
8958        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8959        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8960}
8961
8962/// Where a reviewer seat starts a round: the agent that last answered it when
8963/// it is still on the roster and not marked failed, else the spec's own agent
8964/// unless it failed, else the next roster agent that has not failed, else the
8965/// spec's own agent again (the whole roster failed). Ids no longer on the
8966/// roster are ignored. An empty roster has no handover, so the spec stands.
8967fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8968    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8969        return spec;
8970    };
8971    let ok = |id: &str| !h.failed.contains(id);
8972    if let Some(last) = h.last_ok.as_deref()
8973        && ok(last)
8974        && let Some(s) = roster.iter().find(|s| s.id == last)
8975    {
8976        return s.clone();
8977    }
8978    if ok(&spec.id) {
8979        return spec;
8980    }
8981    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8982    roster
8983        .get(from + 1..)
8984        .into_iter()
8985        .flatten()
8986        .chain(roster.iter())
8987        .find(|s| ok(&s.id))
8988        .cloned()
8989        .unwrap_or(spec)
8990}
8991
8992/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
8993/// seed so a CLI that mints its session id up front (`--session-id`) never
8994/// reuses the uuid the previous agent already opened under the same seat key.
8995pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8996    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8997}
8998
8999/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9000/// so the classifier below and the code that builds the message cannot drift.
9001const TIMED_OUT: &str = "timed out";
9002
9003impl FailClass {
9004    /// `None` for an answer; otherwise how the turn failed.
9005    fn of(out: &AgentOutcome) -> Option<Self> {
9006        match out {
9007            AgentOutcome::Ok(_) => None,
9008            AgentOutcome::Quota(_) => Some(Self::Quota),
9009            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9010            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9011            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9012            // Not a failure of the agent. Callers check for it first; `None`
9013            // here means "no class to hand over on".
9014            AgentOutcome::ResourceBlocked(_) => None,
9015        }
9016    }
9017
9018    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9019    fn stem_word(&self) -> &'static str {
9020        match self {
9021            Self::Quota => "quota",
9022            _ => "handover",
9023        }
9024    }
9025}
9026
9027/// The message's first line with its variable parts removed — digit runs and
9028/// path-like tokens — so "exit code 2" and "exit code 7" read
9029/// as one kind of failure.
9030fn failure_signature(msg: &str) -> String {
9031    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9032    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9033    // the historical machine key while the process diagnosis uses readable prose.
9034    if let Some(code) = line
9035        .strip_prefix("process exited with no usable output (exit code ")
9036        .and_then(|s| s.strip_suffix(')'))
9037        .and_then(|s| s.parse::<i32>().ok())
9038    {
9039        line = format!("exited with some({code}) and no usable output");
9040    } else if matches!(
9041        line.as_str(),
9042        "process exited with no usable output (no exit code reported)"
9043            | "process exited with no usable output (exit code unavailable)"
9044    ) {
9045        line = "exited with none and no usable output".to_owned();
9046    }
9047    let mut out = Vec::new();
9048    for word in line.split_whitespace() {
9049        if word.contains('/') || word.contains('\\') {
9050            out.push("<path>".to_owned());
9051            continue;
9052        }
9053        let mut w = String::new();
9054        let mut in_digits = false;
9055        for c in word.chars() {
9056            if c.is_ascii_digit() {
9057                if !in_digits {
9058                    w.push('#');
9059                }
9060                in_digits = true;
9061            } else {
9062                in_digits = false;
9063                w.push(c);
9064            }
9065        }
9066        out.push(w);
9067    }
9068    out.join(" ").chars().take(120).collect()
9069}
9070
9071/// Whether a seat that just failed with `cur` may go to the next roster agent.
9072/// A quota or a timeout always may. Any other failure may not when the agent
9073/// before it failed the same way: an error the prompt causes would otherwise
9074/// walk the whole roster. `prev` is the class of the immediately preceding
9075/// agent's failure, so a quota or timeout in between breaks the run of
9076/// identical failures by itself.
9077fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9078    match cur {
9079        FailClass::Quota | FailClass::Timeout => true,
9080        FailClass::Other(_) => prev != Some(cur),
9081    }
9082}
9083
9084/// A short human reason for a failed outcome, for the handover record.
9085fn fail_reason(out: &AgentOutcome) -> String {
9086    match out {
9087        AgentOutcome::Ok(_) => String::new(),
9088        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9089        AgentOutcome::Dropped(o) => format!(
9090            "the CLI dropped the stream ({})",
9091            o.dropped
9092                .as_ref()
9093                .map(|d| d.why.as_str())
9094                .unwrap_or("it ended without delivering its answer")
9095        ),
9096        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9097            e.lines().next().unwrap_or("").chars().take(160).collect()
9098        }
9099    }
9100}
9101
9102/// Note one handover in the run: the structured record and, in the timeline,
9103/// the sentence a person reads. A quota keeps the wording it always had.
9104pub(crate) fn record_handover(
9105    state: &mut RunState,
9106    node: &str,
9107    seat: &str,
9108    from: &str,
9109    to: &str,
9110    class: &FailClass,
9111    reason: &str,
9112) {
9113    let message = if *class == FailClass::Quota {
9114        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9115    } else {
9116        format!("{seat}: handed over {from} -> {to} ({reason})")
9117    };
9118    state.event(node, message);
9119    state.handovers.push(Handover {
9120        at: Timestamp::now(),
9121        node: node.to_owned(),
9122        seat: seat.to_owned(),
9123        from: from.to_owned(),
9124        to: to.to_owned(),
9125        reason: reason.to_owned(),
9126    });
9127}
9128
9129/// Did this reply report running a command whose own CLI never confirmed an
9130/// exit status?
9131///
9132/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9133/// command *finished* (see that type's own doc), so this can only be `true`
9134/// for a command whose completion event carried no readable exit code — not
9135/// for one that simply is not mentioned at all. That is the one signal this
9136/// crate can read, from the same record `state.jobs` renders, about a reply
9137/// standing next to work its own CLI cannot vouch for finishing; it is
9138/// deliberately not a check on the exit code's *value* (a fixer legitimately
9139/// runs a command that fails mid-iteration before it succeeds) and not a
9140/// guess at a command still running in the background (which emits no event
9141/// at all, and so leaves no evidence here to find).
9142fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9143    commands.iter().any(|c| c.exit_code.is_none())
9144}
9145
9146/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9147/// trusted as a verified no-op — the adoption guard's own text-level half.
9148///
9149/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9150/// timed out): a marker only earns the benefit of the doubt from a turn the
9151/// CLI itself vouches for finishing properly, the same house style
9152/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9153/// *fix* report to for `commands`. A candidate that timed out, exited
9154/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9155/// is, whatever prose it wrote — this returns `None` before it ever looks at
9156/// `text`. The remaining guards (the tree really is empty, the evidence is
9157/// non-empty) are the caller's: this only reads what the reply *claimed*.
9158fn verified_noop_claim(
9159    usable: bool,
9160    commands: &[agent::CommandEvidence],
9161    text: &str,
9162) -> Option<String> {
9163    (usable && !has_unconfirmed_command(commands))
9164        .then(|| verdict::verified_noop(text))
9165        .flatten()
9166}
9167
9168fn short(commit: &str) -> String {
9169    commit.chars().take(7).collect()
9170}
9171
9172fn make_executable(path: &Path) -> Result<()> {
9173    #[cfg(unix)]
9174    {
9175        use std::os::unix::fs::PermissionsExt as _;
9176        let mut perms = std::fs::metadata(path)?.permissions();
9177        perms.set_mode(0o755);
9178        std::fs::set_permissions(path, perms)?;
9179    }
9180    #[cfg(not(unix))]
9181    {
9182        let _ = path;
9183    }
9184    Ok(())
9185}
9186
9187/// What every seat in one batch shares: where the answers are attributed, the
9188/// prompt overlay they inherit, and the build cache they are told to use.
9189///
9190/// A struct rather than four more parameters: `wave` also needs the run's
9191/// state (to record who is answering right now) and the attempt number, and
9192/// eight positional arguments is both unreadable and a clippy error.
9193struct WaveCtx<'a> {
9194    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9195    /// paid for it.
9196    run: &'a str,
9197    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9198    node: &'a str,
9199    prompts: &'a Prompts,
9200    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9201    cache: Option<&'a Path>,
9202    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9203    /// `JobRecord::round`. `None` for every other node.
9204    round: Option<usize>,
9205    /// Carry each seat's failed-agent history across waves (the review loop
9206    /// only): start-of-round priority and handover choice read
9207    /// [`RunState::seat_history`], and every answer or failure writes it.
9208    carry_seats: bool,
9209    /// The run's park / interrupt requests, so a wait for a shared resource
9210    /// can be abandoned the moment the operator asks to park.
9211    pauses: Option<(&'a Pause, &'a Pause)>,
9212}
9213
9214/// Shared scheduling policy for operations and spawned seat invocations.
9215fn repository_wait(
9216    state: &RunState,
9217    pauses: Option<(&Pause, &Pause)>,
9218) -> crate::repo_lock::WaitPolicy {
9219    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9220    let inherited = crate::repo_lock::WaitPolicy::current();
9221    crate::repo_lock::WaitPolicy::new(
9222        Duration::from_secs(state.config.graph.timeout_resource_wait),
9223        move || {
9224            inherited.cancelled()
9225                || pauses
9226                    .as_ref()
9227                    .is_some_and(|(a, b)| a.parked() || b.parked())
9228        },
9229    )
9230}
9231
9232fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9233    if error.is::<crate::repo_lock::ResourceBlocked>() {
9234        state.resource_block = Some(crate::run::ResourceBlock {
9235            node: "repository".into(),
9236            kind: if error.to_string().contains("disk pressure") {
9237                "disk pressure"
9238            } else {
9239                "repository coordination"
9240            }
9241            .into(),
9242            detail: error.to_string(),
9243            at: Timestamp::now(),
9244        });
9245        state.event("resource", error.to_string());
9246    }
9247}
9248
9249async fn coordinate_operation(
9250    state: &mut RunState,
9251    cwd: &Path,
9252    wait: &crate::repo_lock::WaitPolicy,
9253) -> Result<Option<crate::repo_lock::SeatLease>> {
9254    let result =
9255        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9256            .await;
9257    if let Err(e) = &result {
9258        record_repository_block(state, e);
9259    }
9260    result
9261}
9262
9263/// Run one job, honouring the parallelism budget.
9264async fn run_one(
9265    job: SeatJob,
9266    sem: Arc<Semaphore>,
9267    ctx: &WaveCtx<'_>,
9268    state: &mut RunState,
9269    attempt: usize,
9270) -> (SeatState, AgentOutcome) {
9271    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9272        .await
9273        .pop()
9274        .expect("one job in, one result out");
9275    (seat, out)
9276}
9277
9278/// Run every job concurrently, capped by the semaphore, preserving order.
9279///
9280/// Every seat in the batch is recorded into [`RunState::active`] before the
9281/// wave starts and cleared as each answer lands, so the run's own record says
9282/// who is still being waited on rather than only who finished.
9283async fn wave(
9284    jobs: Vec<SeatJob>,
9285    sem: Arc<Semaphore>,
9286    ctx: &WaveCtx<'_>,
9287    state: &mut RunState,
9288    attempt: usize,
9289) -> Vec<(usize, SeatState, AgentOutcome)> {
9290    let WaveCtx {
9291        run,
9292        node,
9293        prompts,
9294        cache,
9295        round,
9296        carry_seats: _,
9297        pauses,
9298    } = *ctx;
9299    for job in &jobs {
9300        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9301    }
9302    if let Err(e) = state.save() {
9303        // A failed persist of "who is answering right now" must not abort the
9304        // wave: the seats are already being asked, and the alternative is
9305        // losing the answers to save a status line nobody may even be
9306        // watching.
9307        tracing::warn!("could not persist in-progress seats: {e:#}");
9308    }
9309    // Hold the shared build cache's lease for the whole batch, not per job:
9310    // several candidates (an implement wave) or a fixer legitimately share
9311    // one cache concurrently within this run, and that stays untouched — a
9312    // single lease taken once for the whole wave and released once it is
9313    // done is what stops a *different* borrower (another run's own wave, its
9314    // e2e/gate, a human's `magi review`) from interleaving a build into the
9315    // same directory while this one is in flight. Best-effort, not
9316    // all-or-nothing: a wave that cannot get the lease within its own
9317    // longest job's budget still runs — an hour of paid implementer calls is
9318    // not thrown away over cache contention — but every write-allowed seat
9319    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9320    // below), the same fallback a read-only seat always gets, rather than
9321    // building into a directory this run was never granted. The identity
9322    // record is still invalidated below either way, so the next tracked
9323    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9324    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9325    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9326    // never borrowed from a job's agent budget (and never charged to it either,
9327    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9328    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9329    let cache_guard = if let Some(cache_dir) = cache {
9330        if jobs_had_a_writer {
9331            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9332            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9333            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9334            {
9335                Ok(Some(g)) => Some(g),
9336                // Parking: leave without spawning anything. The run parks at
9337                // the next boundary and the seats are asked afresh on resume.
9338                Ok(None) => {
9339                    state.resource_block = Some(crate::run::ResourceBlock {
9340                        node: node.to_owned(),
9341                        kind: "shared build cache".to_owned(),
9342                        detail: "the run is parking".to_owned(),
9343                        at: Timestamp::now(),
9344                    });
9345                    return block_wave(
9346                        jobs,
9347                        state,
9348                        node,
9349                        attempt,
9350                        "parked while waiting for the shared build cache",
9351                    );
9352                }
9353                // An unreadable lease is not contention: keep the old
9354                // fallback (no shared cache for this wave's seats).
9355                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9356                Err(e) => {
9357                    // Writing seats may not build into a cache this run does
9358                    // not hold, and an agent that cannot be given one is not
9359                    // asked at all: the run stops resumably instead.
9360                    let why = format!("{e:#}");
9361                    state.resource_block = Some(crate::run::ResourceBlock {
9362                        node: node.to_owned(),
9363                        kind: "shared build cache".to_owned(),
9364                        detail: why.clone(),
9365                        at: Timestamp::now(),
9366                    });
9367                    state.event(
9368                        "resource",
9369                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9370                    );
9371                    return block_wave(jobs, state, node, attempt, &why);
9372                }
9373            }
9374        } else {
9375            None
9376        }
9377    } else {
9378        None
9379    };
9380    // Every job keeps its whole budget: the wait above was not the agent's.
9381    // Restart the seats' clocks so "started" means "being spawned now".
9382    state.seats_spawning(node, attempt);
9383    if let Err(e) = state.save() {
9384        tracing::warn!("could not persist the start of a wave: {e:#}");
9385    }
9386    // Queueing behind this wave's own seats for a parallelism slot is normal
9387    // (each of them is bounded by its own agent timeout); only a wait beyond
9388    // all of them plus the resource limit means the slots never came back.
9389    let slot_limit = resource_limit
9390        + jobs
9391            .iter()
9392            .map(|j| j.timeout)
9393            .max()
9394            .unwrap_or_default()
9395            .saturating_mul(jobs.len() as u32);
9396    let mut set = tokio::task::JoinSet::new();
9397    // Seats announce themselves here the moment they hold a parallelism
9398    // permit, i.e. just before their CLI is spawned.
9399    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9400    let overlay = prompts.overlay(node);
9401    let repository_wait = repository_wait(state, pauses);
9402    for (i, mut job) in jobs.into_iter().enumerate() {
9403        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9404        if cache.is_some() {
9405            job.prompt.push('\n');
9406            job.prompt
9407                .push_str(&prompt::build_cache_note(node, job.allow_write));
9408        }
9409        let sem = Arc::clone(&sem);
9410        let run = run.to_owned();
9411        let node = node.to_owned();
9412        // Only implementers were told about the task's attachments, so only
9413        // their seats get the directory widened for reading.
9414        let attachments = if node == "implement" {
9415            state.attachments.clone()
9416        } else {
9417            Vec::new()
9418        };
9419        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9420        // `prompt::build_cache_note`'s doc for why setting it anyway is
9421        // exactly how a sandboxed reviewer's write refusal got reported as a
9422        // defect in the patch, not a property of its own seat. And a
9423        // write-allowed one is handed it only when the lease above was
9424        // actually acquired: a wave that could not get it (`cache_guard` is
9425        // `None`, see its own comment) must not send seats to build into a
9426        // directory this run does not hold - that is the exact concurrent,
9427        // unmanaged-write race this module exists to prevent, not something
9428        // "proceeding anyway" is allowed to reintroduce.
9429        let cache = cache
9430            .filter(|_| job.allow_write && cache_guard.is_some())
9431            .map(Path::to_path_buf);
9432        let spawned_tx = spawned_tx.clone();
9433        let repository_wait = repository_wait.clone();
9434        let coordination = state.config.coordination.clone();
9435        let fetched_base = state.fetched_base.clone();
9436        set.spawn(async move {
9437            // A wait for a parallelism slot is scheduling, not the agent: it
9438            // is bounded by its own limit and ends as a resource block, never
9439            // as a timeout of the agent that was not started.
9440            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9441                Ok(p) => p,
9442                Err(_) => {
9443                    return (
9444                        i,
9445                        job.seat,
9446                        AgentOutcome::ResourceBlocked(format!(
9447                            "no parallel slot became free within {}s",
9448                            slot_limit.as_secs()
9449                        )),
9450                        false,
9451                    );
9452                }
9453            };
9454            spawned_tx.send(job.seat.key.clone()).ok();
9455            let mut seat = job.seat;
9456            let out = repository_wait
9457                .scope(agent::invoke(
9458                    &job.spec,
9459                    &mut seat,
9460                    &Invocation {
9461                        scope: crate::agent::InvocationScope::Repository,
9462                        coordination: coordination.as_ref(),
9463                        fetched_base: fetched_base.as_ref(),
9464                        cwd: &job.cwd,
9465                        prompt: &job.prompt,
9466                        timeout: job.timeout,
9467                        allow_write: job.allow_write,
9468                        unsandboxed: false,
9469                        sessions: job.sessions,
9470                        artifacts: &job.artifacts,
9471                        stem: &job.stem,
9472                        run: &run,
9473                        node: &node,
9474                        cache_dir: cache.as_deref(),
9475                        attachments: &attachments,
9476                        writable: &[],
9477                    },
9478                ))
9479                .await;
9480            let interrupted = out
9481                .as_ref()
9482                .err()
9483                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9484            let out = match out {
9485                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9486                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9487                // Billed work the CLI failed to hand over is not an ordinary
9488                // failure, but its text is the CLI's raw error JSON, not an
9489                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9490                // read it as one by forgetting to check. `usable()` is always
9491                // false here (dropped implies an empty response), so this has
9492                // to be checked before the catch-all `Failed` below or the
9493                // one shape this exists for is lost with the rest.
9494                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9495                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9496                Ok(o) => AgentOutcome::Failed(format!(
9497                    "process exited with no usable output ({})",
9498                    crate::notices::exit_status(o.exit_code)
9499                )),
9500                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9501                    AgentOutcome::ResourceBlocked(e.to_string())
9502                }
9503                Err(e) => AgentOutcome::Failed(e.to_string()),
9504            };
9505            (i, seat, out, interrupted)
9506        });
9507    }
9508    drop(spawned_tx);
9509    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9510    loop {
9511        let joined = tokio::select! {
9512            Some(key) = spawned_rx.recv() => {
9513                // The clock of this seat starts now, not when it queued.
9514                state.seat_spawned(&key);
9515                if let Err(e) = state.save() {
9516                    tracing::warn!("could not persist a seat's start: {e:#}");
9517                }
9518                continue;
9519            }
9520            joined = set.join_next() => joined,
9521        };
9522        let Some(joined) = joined else {
9523            break;
9524        };
9525        let (i, seat, out, interrupted) = match joined {
9526            Ok(v) => v,
9527            // No seat to clear: a panicked task never reported which one it
9528            // was. The defensive sweep below this loop is what stops that
9529            // seat's `active` entry from surviving forever.
9530            Err(e) => {
9531                tracing::error!("agent task panicked: {e}");
9532                continue;
9533            }
9534        };
9535        state.seat_finished(&seat.key);
9536        if interrupted
9537            && node == "implement"
9538            && let Some(label) = state
9539                .candidates
9540                .iter()
9541                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9542                .map(|candidate| candidate.label)
9543        {
9544            state.candidate_preservation.insert(
9545                label,
9546                "interrupted implementation awaiting persistence".into(),
9547            );
9548            state.interrupted_implementations.insert(label);
9549        }
9550        if let AgentOutcome::ResourceBlocked(why) = &out
9551            && state.resource_block.is_none()
9552        {
9553            state.resource_block = Some(crate::run::ResourceBlock {
9554                node: node.to_owned(),
9555                kind: if why.contains("disk pressure") {
9556                    "disk pressure"
9557                } else if why.contains("repository coordination") {
9558                    "repository coordination"
9559                } else {
9560                    "parallel slot"
9561                }
9562                .to_owned(),
9563                detail: why.clone(),
9564                at: Timestamp::now(),
9565            });
9566            let activity = if interrupted {
9567                "agent execution interrupted"
9568            } else {
9569                "no agent was started"
9570            };
9571            state.event("resource", format!("{node}: {activity} - {why}"));
9572        }
9573        record_jobs(state, node, round, &seat.key, &out);
9574        if let Err(e) = state.save() {
9575            tracing::warn!("could not persist a seat's completion: {e:#}");
9576        }
9577        if collected.len() <= i {
9578            collected.resize_with(i + 1, || None);
9579        }
9580        collected[i] = Some((i, seat, out));
9581    }
9582    // Belt-and-braces for the panic branch above: every seat this exact batch
9583    // started shares this `(node, attempt)` pair, and every seat that finished
9584    // normally already cleared itself, so anything left tagged with it here
9585    // can only be a panicked task's leftover. Cleared unconditionally rather
9586    // than left to read as still answering forever.
9587    if state
9588        .active
9589        .values()
9590        .any(|a| a.node == node && a.attempt == attempt)
9591    {
9592        state
9593            .active
9594            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9595        if let Err(e) = state.save() {
9596            tracing::warn!("could not persist the end of a wave: {e:#}");
9597        }
9598    }
9599    // Whether or not the lease above was actually held, several worktrees
9600    // may just have built into the cache with nothing here able to name one
9601    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9602    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9603    // clean it might not have strictly needed; trusting a stale match would
9604    // cost it a wrong answer.
9605    if let Some(cache_dir) = cache
9606        && jobs_had_a_writer
9607    {
9608        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9609    }
9610    if let Some(guard) = cache_guard {
9611        guard.release();
9612    }
9613    collected.into_iter().flatten().collect()
9614}
9615
9616/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9617/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9618/// this is data collection, not the fix-specific completion contract in
9619/// [`Runner::continue_fix_report`], and applies regardless of which node
9620/// asked.
9621///
9622/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9623/// evidence from; `Failed` does not, and correctly contributes nothing — a
9624/// timeout or crash is not itself evidence about a command the seat may have
9625/// started.
9626fn record_jobs(
9627    state: &mut RunState,
9628    node: &str,
9629    round: Option<usize>,
9630    seat: &str,
9631    out: &AgentOutcome,
9632) {
9633    let commands: &[agent::CommandEvidence] = match out {
9634        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9635        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9636    };
9637    let checked_at = Timestamp::now();
9638    for c in commands {
9639        state.jobs.push(JobRecord {
9640            node: node.to_owned(),
9641            round,
9642            seat: seat.to_owned(),
9643            id: c.id.clone(),
9644            description: c.description.clone(),
9645            checked_at,
9646            status: match c.exit_code {
9647                Some(0) => JobStatus::Completed,
9648                Some(_) => JobStatus::Failed,
9649                None => JobStatus::Unknown,
9650            },
9651            exit_code: c.exit_code,
9652            result_summary: c.result_summary.clone(),
9653            source: c.source.clone(),
9654        });
9655    }
9656}
9657
9658/// Is a review round clean, given how many reviewer seats answered against
9659/// how many the round expected?
9660///
9661/// A seat that never answered (timeout, crash, unparsable output) is not a
9662/// seat that read the patch and found nothing — treating it as such is
9663/// exactly the bug this function exists to close. Under the default `block`
9664/// policy a missing seat can never be clean; `warn` still requires the seats
9665/// that *did* answer to have found nothing blocking and verification to be
9666/// green.
9667///
9668/// `quota_missing` narrows that `block` default for exactly one cause of
9669/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9670/// a session limit lifts by the very next round buys nothing — the seat is
9671/// asked again with the same quota — so once every missing seat is accounted
9672/// for by a quota loss (and at least one seat *did* answer, so a decision has
9673/// something to rest on) the round is decided on the panel that could answer,
9674/// same as `warn` would. A panel that lost every seat to quota is not
9675/// decided here: `answered == 0` falls through to the existing `block`
9676/// fallback so a fully collapsed panel still waits rather than landing on no
9677/// review at all.
9678fn round_is_clean(
9679    blocking: usize,
9680    e2e_ok: bool,
9681    answered: usize,
9682    expected: usize,
9683    quota_missing: usize,
9684    policy: IncompleteReviewPolicy,
9685) -> bool {
9686    if blocking != 0 || !e2e_ok {
9687        return false;
9688    }
9689    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9690        return true;
9691    }
9692    answered > 0 && expected - answered <= quota_missing
9693}
9694
9695/// The review loop's own conclusion, derived entirely from its persisted
9696/// round records and the round budget that produced them — never from
9697/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9698/// recomputes the identical answer regardless of what an earlier node in the
9699/// same walk, or a previous walk, did to `status`.
9700///
9701/// `None` while more rounds remain to try, including when review never ran
9702/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9703/// gone clean, or the budget is spent, or the tree has stopped moving (see
9704/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9705///
9706/// - An incomplete panel that raised nothing is missing input, not a
9707///   verified tree — never a hand-off candidate, whatever verification said
9708///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9709/// - Otherwise, green e2e on the last round hands off (see
9710///   [`Runner::stop_reviewing`]); red e2e blocks.
9711///
9712/// A last round whose own verification is still `ResourceBlocked` — magi
9713/// itself never got a command to run, not evidence the patch is broken —
9714/// is neither: this returns `None` for it too, the same as "more rounds
9715/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9716/// handling of that shape) instead of this cheap recomputation guessing a
9717/// verdict a real attempt never produced.
9718fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9719    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9720        return Some(RunStatus::Gating);
9721    }
9722    let last = reviews.last()?;
9723    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9724    if reviews.len() < max_rounds && !stagnant {
9725        return None;
9726    }
9727    if last.incomplete() && last.blocking == 0 {
9728        return Some(RunStatus::Blocked);
9729    }
9730    if last.e2e_status() == E2eStatus::ResourceBlocked {
9731        return None;
9732    }
9733    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9734        RunStatus::Gating
9735    } else {
9736        RunStatus::Blocked
9737    })
9738}
9739
9740/// How long a re-ask may take, given the budget the first attempt had.
9741///
9742/// A `nudged` retry is a request to restate an answer the seat has already
9743/// worked out: it carries no new work, so it does not deserve the original
9744/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9745/// 133 seconds while a third sat for over ten minutes on a resumed session
9746/// holding 410 KB of prior output - and because the retry had inherited the
9747/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9748/// judging round whose other seats were long finished.
9749///
9750/// A quarter of the budget, with a floor so that a deliberately short timeout
9751/// does not collapse to nothing. A retry that re-sends the whole prompt
9752/// (because the seat kept no context) is the original job again, and keeps the
9753/// original budget.
9754pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9755    if nudged {
9756        (full / 4).max(Duration::from_secs(120)).min(full)
9757    } else {
9758        full
9759    }
9760}
9761
9762/// Run a wave and parse each reply, re-asking the seats whose reply was
9763/// unusable.
9764///
9765/// The re-ask is a nudge rather than the whole prompt again when the seat still
9766/// holds its conversation, which is the difference between a cheap retry and
9767/// paying for the entire candidate set twice.
9768///
9769/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9770/// successor in `roster` is handed to it instead of being re-asked: the
9771/// handover is the retry. A seat with no successor left (a single-agent
9772/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9773/// the asks to one seat in one node number at most
9774/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9775/// is asked once (a dropped stream is nudged first), and only the last agent
9776/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9777/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9778/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9779/// other failure stops the chain when the previous agent failed the same way
9780/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9781/// [`has_context`] is false and the job's own full prompt and full budget are
9782/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9783/// intermediate ones are not losses) and is returned as a failure like any
9784/// other absent seat — the caller decides whether the panel still has a
9785/// quorum. An empty `roster` disables handover: failures are nudged as they
9786/// always were, and a quota is simply lost. A reply that fails to parse or
9787/// validate is the prompt's doing and is only ever nudged, never handed over.
9788///
9789/// The returned [`SeatState`] names the agent that answered (or tried last).
9790#[allow(clippy::too_many_arguments)]
9791async fn ask_json_wave<T>(
9792    jobs: Vec<SeatJob>,
9793    sem: Arc<Semaphore>,
9794    retries: usize,
9795    roster: &[AgentSpec],
9796    ctx: &WaveCtx<'_>,
9797    losses: &mut Vec<QuotaLoss>,
9798    state: &mut RunState,
9799    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9800) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9801where
9802    T: serde::de::DeserializeOwned + Send + 'static,
9803{
9804    ask_wave_with(
9805        jobs,
9806        sem,
9807        retries,
9808        roster,
9809        ctx,
9810        losses,
9811        state,
9812        &|text: &str| {
9813            let v = verdict::extract_json::<T>(text)?;
9814            validate(&v)?;
9815            Ok(v)
9816        },
9817    )
9818    .await
9819}
9820
9821/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9822/// a node whose answer is prose (deliberation) shares the same handover,
9823/// failure classification, quota bookkeeping and bounds instead of a copy.
9824///
9825/// A seat handed to another roster agent is sent the job's `handover` prompt
9826/// (when it has one) rather than `prompt`: the new agent has no session, so a
9827/// resume-style prompt would be incomplete. That holds for the handover ask
9828/// and for every nudge to that agent whose `has_context` is false.
9829#[allow(clippy::too_many_arguments)]
9830async fn ask_wave_with<T>(
9831    jobs: Vec<SeatJob>,
9832    sem: Arc<Semaphore>,
9833    retries: usize,
9834    roster: &[AgentSpec],
9835    ctx: &WaveCtx<'_>,
9836    losses: &mut Vec<QuotaLoss>,
9837    state: &mut RunState,
9838    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9839) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9840where
9841    T: Send + 'static,
9842{
9843    let n = jobs.len();
9844    let originals: Vec<SeatJob> = jobs;
9845    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9846    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9847    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9848    // answer, N once it has gone through N nudges. Read back once this
9849    // returns, so a caller building a history record (`ReviewRecord`) can
9850    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9851    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9852    // field's own doc.
9853    let mut nudges: Vec<usize> = vec![0; n];
9854    // The agent now occupying each seat, the ids it has already been through,
9855    // where in the roster the walk began, the class of the last failure, and
9856    // the stem word of a handover not yet asked (full prompt, full budget).
9857    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9858    let mut tried: Vec<BTreeSet<String>> = specs
9859        .iter()
9860        .map(|s| BTreeSet::from([s.id.clone()]))
9861        .collect();
9862    let starts: Vec<usize> = specs
9863        .iter()
9864        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9865        .collect();
9866    let carry = ctx.carry_seats && !roster.is_empty();
9867    // Carried across rounds: ids that failed the seat earlier, and how the
9868    // last failure went (so a repeat of it is not handed over again).
9869    let carried: Vec<BTreeSet<String>> = originals
9870        .iter()
9871        .map(|j| {
9872            state
9873                .seat_history
9874                .get(&j.seat.key)
9875                .filter(|_| carry)
9876                .map(|h| h.failed.clone())
9877                .unwrap_or_default()
9878        })
9879        .collect();
9880    let mut prev: Vec<Option<FailClass>> = originals
9881        .iter()
9882        .map(|j| {
9883            state
9884                .seat_history
9885                .get(&j.seat.key)
9886                .filter(|_| carry)
9887                .and_then(|h| h.last_fail.clone())
9888        })
9889        .collect();
9890    let next_agent =
9891        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9892            let others: BTreeSet<String> = specs
9893                .iter()
9894                .enumerate()
9895                .filter(|(j, _)| *j != i)
9896                .map(|(_, s)| s.id.clone())
9897                .collect();
9898            pick_successor(
9899                roster,
9900                starts[i],
9901                tried,
9902                carry.then(|| &carried[i]),
9903                &others,
9904            )
9905            .cloned()
9906        };
9907    let mut fresh: Vec<Option<String>> = vec![None; n];
9908    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9909    let mut pending: Vec<usize> = (0..n).collect();
9910
9911    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
9912    // asks: an agent with a successor is asked once and handed over, and only
9913    // a seat with no successor spends `retries` nudges on the same agent. This
9914    // only guarantees the loop's own termination whatever those say.
9915    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9916    for round in 0..max_rounds {
9917        if pending.is_empty() {
9918            break;
9919        }
9920        let mut batch = Vec::with_capacity(pending.len());
9921        let mut renudged: Vec<&str> = Vec::new();
9922        for &i in &pending {
9923            let src = &originals[i];
9924            // A seat now held by another agent than the job named has no
9925            // session of its own: it gets the full-context prompt whenever
9926            // it is asked in full (the handover ask, a nudge it cannot
9927            // resume).
9928            let full: &str = match &src.handover {
9929                Some(h) if specs[i].id != src.spec.id => h,
9930                _ => &src.prompt,
9931            };
9932            // The prompt and the budget are one decision: a nudge restates
9933            // finished work, a re-sent prompt redoes it.
9934            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9935                (
9936                    full.to_owned(),
9937                    src.timeout,
9938                    format!("{}-{word}-{}", src.stem, specs[i].id),
9939                )
9940            } else if nudges[i] == 0 {
9941                (full.to_owned(), src.timeout, src.stem.clone())
9942            } else {
9943                renudged.push(src.seat.key.as_str());
9944                let why = done[i]
9945                    .as_ref()
9946                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
9947                    .unwrap_or_else(|| "no parsable answer".to_owned());
9948                let nudge = prompt::nudge(&why);
9949                let nudged = has_context(&specs[i], &seats[i], src.sessions);
9950                let prompt = if nudged {
9951                    nudge
9952                } else {
9953                    format!("{full}\n\n---\n\n{nudge}")
9954                };
9955                (
9956                    prompt,
9957                    retry_budget(src.timeout, nudged),
9958                    format!("{}-retry{}", src.stem, nudges[i]),
9959                )
9960            };
9961            batch.push(SeatJob {
9962                spec: specs[i].clone(),
9963                seat: seats[i].clone(),
9964                cwd: src.cwd.clone(),
9965                prompt,
9966                timeout,
9967                allow_write: src.allow_write,
9968                sessions: src.sessions,
9969                artifacts: src.artifacts.clone(),
9970                stem,
9971                handover: None,
9972            });
9973        }
9974
9975        if !renudged.is_empty() {
9976            state.event(
9977                ctx.node,
9978                format!("retry {round}: re-asking {}", renudged.join(", ")),
9979            );
9980        }
9981        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9982        let mut still = Vec::new();
9983        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9984            seats[i] = seat;
9985            // No agent was asked: nothing to hand over, nudge, remember or
9986            // charge. The caller sees `resource_block` and stops the node.
9987            if let AgentOutcome::ResourceBlocked(why) = &out {
9988                done[i] = Some(Err(anyhow::anyhow!("{why}")));
9989                continue;
9990            }
9991            let class = FailClass::of(&out);
9992            // A dropped stream is nudged first (the conversation is still
9993            // there to pick up); only a seat whose nudges are spent hands over.
9994            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9995                && nudges[i] < retries
9996                && !roster.is_empty();
9997            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9998                let next = should_hand_over(prev[i].as_ref(), &cur)
9999                    .then(|| next_agent(i, &tried[i], &specs))
10000                    .flatten();
10001                if carry {
10002                    let h = state
10003                        .seat_history
10004                        .entry(originals[i].seat.key.clone())
10005                        .or_default();
10006                    h.failed.insert(specs[i].id.clone());
10007                    h.last_fail = Some(cur.clone());
10008                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10009                        h.last_ok = None;
10010                    }
10011                    // Saved before the next agent is asked, so a restart in
10012                    // between does not forget who failed.
10013                    if let Err(e) = state.save() {
10014                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10015                    }
10016                }
10017                if let Some(next) = next {
10018                    record_handover(
10019                        state,
10020                        ctx.node,
10021                        &originals[i].seat.key,
10022                        &specs[i].id,
10023                        &next.id,
10024                        &cur,
10025                        &fail_reason(&out),
10026                    );
10027                    tried[i].insert(next.id.clone());
10028                    prev[i] = Some(cur.clone());
10029                    seats[i] =
10030                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10031                    specs[i] = next;
10032                    fresh[i] = Some(cur.stem_word().to_owned());
10033                    nudges[i] = 0;
10034                    done[i] = Some(Err(anyhow::anyhow!(
10035                        "handed over after: {}",
10036                        fail_reason(&out)
10037                    )));
10038                    still.push(i);
10039                    continue;
10040                }
10041            }
10042            if carry
10043                && !nudge_first
10044                && let Some(cur) = class.clone()
10045            {
10046                // The chain ended here (no successor, or a repeated failure).
10047                let h = state
10048                    .seat_history
10049                    .entry(originals[i].seat.key.clone())
10050                    .or_default();
10051                h.failed.insert(specs[i].id.clone());
10052                h.last_fail = Some(cur);
10053            }
10054            let parsed = match out {
10055                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10056                AgentOutcome::Quota(o) => {
10057                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10058                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10059                }
10060                // Not a parseable answer: the nudge loop re-asks it, which is
10061                // exactly what a dropped stream needs. Just don't hand its raw
10062                // error JSON to `extract_json`.
10063                AgentOutcome::Dropped(o) => {
10064                    let why = o
10065                        .dropped
10066                        .as_ref()
10067                        .map(|d| d.why.as_str())
10068                        .unwrap_or("the CLI ended the stream without delivering its answer");
10069                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10070                }
10071                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10072                    Err(anyhow::anyhow!(e))
10073                }
10074            };
10075            let quota = class == Some(FailClass::Quota);
10076            let failed = parsed.is_err();
10077            done[i] = Some(parsed);
10078            if carry && !failed {
10079                let h = state
10080                    .seat_history
10081                    .entry(originals[i].seat.key.clone())
10082                    .or_default();
10083                h.failed.remove(&specs[i].id);
10084                h.last_ok = Some(specs[i].id.clone());
10085                h.last_fail = None;
10086            }
10087            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10088            // fail the same way; and never re-ask a seat that already parsed.
10089            // A failed agent that still has a successor is not re-asked
10090            // either: the handover was its remedy and has just been refused
10091            // (the chain stops on a repeated failure class). A seat with no
10092            // successor left (a single-agent roster, the roster's tail, or an
10093            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10094            let agent_failure = class.is_some()
10095                && !nudge_first
10096                && !roster.is_empty()
10097                && next_agent(i, &tried[i], &specs).is_some();
10098            if failed && !quota && !agent_failure && nudges[i] < retries {
10099                nudges[i] += 1;
10100                still.push(i);
10101            }
10102        }
10103        pending = still;
10104    }
10105
10106    // One loss per seat whose chain ended on a quota: the intermediate ones
10107    // were absorbed by a handover and are not losses.
10108    for (i, q) in last_quota.into_iter().enumerate() {
10109        if let Some(reset) = q {
10110            losses.push(QuotaLoss {
10111                seat: originals[i].seat.key.clone(),
10112                node: ctx.node.to_owned(),
10113                at: Timestamp::now(),
10114                reset,
10115            });
10116        }
10117    }
10118
10119    seats
10120        .into_iter()
10121        .zip(done)
10122        .zip(nudges)
10123        .map(|((seat, res), attempts)| {
10124            (
10125                seat,
10126                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10127                attempts,
10128            )
10129        })
10130        .collect()
10131}
10132
10133/// Did the wave end without asking any agent (see [`block_wave`])?
10134fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10135    results
10136        .iter()
10137        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10138}
10139
10140/// Settle a wave that never spawned anything: every job answers
10141/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10142/// else about the seat (session, history, quota) is touched.
10143fn block_wave(
10144    jobs: Vec<SeatJob>,
10145    state: &mut RunState,
10146    node: &str,
10147    attempt: usize,
10148    why: &str,
10149) -> Vec<(usize, SeatState, AgentOutcome)> {
10150    state.resource_wait = None;
10151    state
10152        .active
10153        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10154    if let Err(e) = state.save() {
10155        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10156    }
10157    jobs.into_iter()
10158        .enumerate()
10159        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10160        .collect()
10161}
10162
10163/// Acquire the shared build cache's lease, waiting out contention within
10164/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10165/// unbounded wait is never acceptable).
10166///
10167/// A first, non-blocking check happens before ever waiting; if it finds the
10168/// lease busy, that fact is logged as a `verify` event *and* flushed with
10169/// [`RunState::save`] immediately — not only once the wait finally succeeds
10170/// or gives up — so a `magi show` run by a different process while this one
10171/// is still waiting reads a `run.json` that says so, rather than whatever it
10172/// looked like before the wait started. The same applies to the terminal
10173/// failure: logged and saved before this returns `Err`, so a caller that
10174/// could not get the lease at all still leaves a legible record of why.
10175async fn acquire_cache_lease(
10176    state: &mut RunState,
10177    cache_dir: &Path,
10178    owner: &crate::cache::Owner,
10179    budget: Duration,
10180    context: &str,
10181    cancel: &(dyn Fn() -> bool + Sync),
10182) -> Result<Option<crate::cache::Guard>> {
10183    let home = crate::run::home();
10184    let started = Instant::now();
10185    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10186        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10187        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10188        Err(e) => {
10189            state.event(
10190                "verify",
10191                format!("{context}: could not check the shared build cache: {e:#}"),
10192            );
10193            if let Err(e2) = state.save() {
10194                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10195            }
10196            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10197        }
10198    };
10199    state.event(
10200        "verify",
10201        format!(
10202            "{context}: waiting for the shared build cache at {} ({})",
10203            cache_dir.display(),
10204            busy.describe()
10205        ),
10206    );
10207    // Seats of this node are waiting, not running: say so, so the countdown a
10208    // reader sees only ever measures an agent that has been spawned.
10209    state.resource_wait = Some(crate::run::ResourceWait {
10210        node: context.to_owned(),
10211        kind: "shared build cache".to_owned(),
10212        holder: busy.describe(),
10213        since: Timestamp::now(),
10214        limit_secs: budget.as_secs(),
10215    });
10216    if let Err(e) = state.save() {
10217        tracing::warn!("could not persist a cache wait: {e:#}");
10218    }
10219    let remaining = budget.saturating_sub(started.elapsed());
10220    let waited = crate::cache::wait_for_unless(
10221        &home,
10222        cache_dir,
10223        owner,
10224        remaining,
10225        Duration::from_secs(5),
10226        cancel,
10227    )
10228    .await;
10229    state.resource_wait = None;
10230    match waited {
10231        Ok(g) => Ok(g),
10232        Err(e) => {
10233            state.event("verify", format!("{context}: {e:#}"));
10234            if let Err(e2) = state.save() {
10235                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10236            }
10237            Err(e)
10238        }
10239    }
10240}
10241
10242/// The lease state could not be read at all (unreadable lease file, a
10243/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10244/// known to be contended, so a wave degrades to running without a shared cache
10245/// instead of stopping on a hold that [`crate::triage`] could never release.
10246#[derive(Debug)]
10247struct CacheCheckError(String);
10248
10249impl std::fmt::Display for CacheCheckError {
10250    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10251        f.write_str(&self.0)
10252    }
10253}
10254
10255impl std::error::Error for CacheCheckError {}
10256
10257/// Run `body` — a verify command batch — while holding the shared build
10258/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10259/// never interleave with another borrower's build against the same
10260/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10261/// timeout, or a human's own `magi review`. See the `cache` module doc for
10262/// why this matters more than Cargo's own per-target locking covers — two
10263/// *different* worktrees building the same package name/version into one
10264/// cache directory is a staleness bug, not a lock contention one.
10265///
10266/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10267/// agent waves do. Repository scheduling also has its own bounded,
10268/// cancellable resource-wait policy. Contention must not consume the
10269/// verification budget: `body` receives the full `budget` after both leases
10270/// are acquired and the cache is fresh. Without a shared cache, only
10271/// repository coordination is acquired before handing over the full budget.
10272///
10273/// A lease that cannot be acquired within the resource wait is reported as a single
10274/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10275/// verification — the same shape a spawn failure already takes in
10276/// [`run_commands`], so a caller need not special-case it.
10277#[allow(clippy::too_many_arguments)]
10278async fn with_cache_lease<'s, F, Fut>(
10279    state: &'s mut RunState,
10280    cache_dir: Option<&Path>,
10281    node: &str,
10282    seat: &str,
10283    worktree: &Path,
10284    head: &str,
10285    budget: Duration,
10286    context: &str,
10287    wait: &crate::repo_lock::WaitPolicy,
10288    body: F,
10289) -> Result<(Vec<CommandOutcome>, bool)>
10290where
10291    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10292    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10293{
10294    // Match editing waves: cache first, repository second. In particular,
10295    // pre_gate must never hold the repository while waiting for a wave's cache.
10296    let started = Instant::now();
10297    let home = crate::run::home();
10298    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10299    let guard = if let Some(cache_dir) = cache_dir {
10300        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10301        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10302            wait.cancelled()
10303        })
10304        .await
10305        .and_then(|g| {
10306            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10307        }) {
10308            Ok(g) => Some(g),
10309            Err(e) => {
10310                return Ok((
10311                    vec![CommandOutcome {
10312                        command: "(waiting for the shared build cache)".into(),
10313                        code: None,
10314                        output_tail: e.to_string(),
10315                        duration_ms: started.elapsed().as_millis() as u64,
10316                        resource_blocked: true,
10317                    }],
10318                    false,
10319                ));
10320            }
10321        }
10322    } else {
10323        None
10324    };
10325    let _coordination = coordinate_operation(state, worktree, wait).await?;
10326    if let Some(cache_dir) = cache_dir {
10327        let identity = crate::cache::Identity::new(worktree, head);
10328        if let Err(e) = crate::cache::ensure_fresh(
10329            &home,
10330            cache_dir,
10331            &identity,
10332            &state.config.verify.cargo_manifests(),
10333        ) {
10334            state.event(
10335                "verify",
10336                format!(
10337                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10338                    worktree.display(),
10339                    short(head)
10340                ),
10341            );
10342            if let Some(guard) = guard {
10343                guard.release();
10344            }
10345            return Ok((
10346                vec![CommandOutcome {
10347                    command: "(confirming the shared build cache is fresh)".into(),
10348                    code: None,
10349                    output_tail: e.to_string(),
10350                    duration_ms: started.elapsed().as_millis() as u64,
10351                    resource_blocked: true,
10352                }],
10353                false,
10354            ));
10355        }
10356    }
10357    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10358    if !timed_out_pids.is_empty() {
10359        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10360    }
10361    // Drop repository coordination before releasing the cache, reversing the
10362    // acquisition order. A cancelled/error future also drops both guards.
10363    drop(_coordination);
10364    if let Some(guard) = guard {
10365        guard.release();
10366    }
10367    Ok((outcomes, retried))
10368}
10369
10370/// Poll `pids` — commands [`run_commands`] reports as still running when its
10371/// own timeout elapsed — until every one is confirmed gone, or
10372/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10373///
10374/// Real confirmation where confirmation is possible, not a substitute for
10375/// full process-tree observation: a grandchild the timed-out process spawned
10376/// and that survives independently of it is invisible to a pid check the
10377/// same way it always was, and continuing to observe and collect *that*
10378/// stays a different piece of work with its own owner. This only narrows a
10379/// fixed blind wait into an actual check of the pids this process does know
10380/// about.
10381async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10382    wait_for_pids_with(
10383        pids,
10384        crate::proc::pid_alive,
10385        LEASE_RELEASE_POLL,
10386        LEASE_RELEASE_MAX_WAIT,
10387    )
10388    .await;
10389}
10390
10391/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10392/// interval and ceiling supplied by the caller, so the polling *logic* -
10393/// returns as soon as every pid reports dead, gives up at the ceiling
10394/// otherwise - is testable on millisecond durations without asking the real
10395/// OS about a pid at all.
10396async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10397    pids: &[u32],
10398    alive: F,
10399    poll: Duration,
10400    max_wait: Duration,
10401) {
10402    let deadline = Instant::now() + max_wait;
10403    loop {
10404        if pids.iter().all(|&pid| !alive(pid)) {
10405            return;
10406        }
10407        if Instant::now() >= deadline {
10408            return;
10409        }
10410        tokio::time::sleep(poll).await;
10411    }
10412}
10413
10414/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10415/// admission that it could not even get a verify command to run, as opposed
10416/// to evidence the command actually produced? A caller that would otherwise
10417/// read a resource-blocked outcome as a red command must check this first:
10418/// see [`Runner::gate`], which retries rather than records `Blocked` when
10419/// this is true.
10420fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10421    outcomes.iter().any(|o| o.resource_blocked)
10422}
10423
10424/// What [`Runner::gate_fix_round`] decided.
10425enum GateFix {
10426    /// The tree changed and `verify.e2e` is still green: run the gate again.
10427    Retry,
10428    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10429    /// last failure stands and the run ends blocked.
10430    Stop,
10431    /// `verify.e2e` could not run after the fix (magi's own contention):
10432    /// decide nothing now, a later reentry retries.
10433    Defer,
10434}
10435
10436/// Is every red command in `outcomes` an ordinary failure the code could
10437/// explain: it ran, exited non-zero, and said something?
10438///
10439/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10440/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10441/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10442/// what the command is stays the gate's business.
10443fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10444    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10445    red.peek().is_some()
10446        && red.all(|o| {
10447            !o.resource_blocked
10448                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10449                && !o.output_tail.trim().is_empty()
10450        })
10451}
10452
10453/// Describe one verify command's outcome for the event log, distinguishing a
10454/// build/link failure — the toolchain never produced a binary to run — from
10455/// an actual test failure, since only the latter is a verdict on the patch.
10456fn e2e_outcome_label(o: &CommandOutcome) -> String {
10457    if o.ok() {
10458        return "pass".to_owned();
10459    }
10460    let reason = if o.build_failed() {
10461        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10462    } else {
10463        format!("FAIL ({:?})", o.code)
10464    };
10465    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10466}
10467
10468/// Run `verify.e2e`, retrying once if the first attempt could not build or
10469/// link — a build/link failure is frequently a race against a shared
10470/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10471/// `verify` event per command, tagged with `context` (normally `"round N"`)
10472/// so the two call sites that need this — the ordinary per-round leg in
10473/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10474/// it will ever call a round green — read identically in the event log.
10475async fn run_e2e_with_retry(
10476    state: &mut RunState,
10477    shell: &[String],
10478    commands: &[String],
10479    worktree: &Path,
10480    timeout: Duration,
10481    context: &str,
10482) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10483    let (mut e2e, mut timed_out_pids) = run_commands(
10484        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10485    )
10486    .await;
10487    for o in &e2e {
10488        state.event(
10489            "verify",
10490            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10491        );
10492    }
10493    // A build/link failure is not a verdict on the patch — it is frequently a
10494    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10495    // one retry before letting a red like that decide the round.
10496    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10497    if verify_retried {
10498        state.event(
10499            "verify",
10500            format!(
10501                "{context}: verify could not build/link, not a test result — retrying once \
10502                 before concluding"
10503            ),
10504        );
10505        let retried = run_commands(
10506            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10507        )
10508        .await;
10509        e2e = retried.0;
10510        // Both attempts' timeouts matter, not just the last one: the first
10511        // attempt's descendants may still be alive alongside the retry's.
10512        timed_out_pids.extend(retried.1);
10513        for o in &e2e {
10514            state.event(
10515                "verify",
10516                format!(
10517                    "{context}: retry `{}` -> {}",
10518                    o.command,
10519                    e2e_outcome_label(o)
10520                ),
10521            );
10522        }
10523    }
10524    (e2e, verify_retried, timed_out_pids)
10525}
10526
10527/// Run configured shell commands in `cwd`, in order. The second element is
10528/// the pid of every command that hit `timeout` and was still running when
10529/// this stopped waiting on it (best-effort: `None` when the platform did not
10530/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10531/// that releases a shared resource afterward needs to know.
10532///
10533/// Records `task` into [`RunState::active`] at every command boundary
10534/// (`RunState::task_command`) and clears it once the whole list has run
10535/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10536/// for minutes with no seat and no output of its own to show for it (see
10537/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10538/// apart from "not yet run" without this), and this is the only place that
10539/// knows which command is running right now and how many are left. Three
10540/// saves per command — start, not per second — matching the same "only at a
10541/// boundary" rule [`wave`] already follows for seats.
10542#[allow(clippy::too_many_arguments)]
10543async fn run_commands(
10544    state: &mut RunState,
10545    node: &str,
10546    task: &str,
10547    attempt: usize,
10548    shell: &[String],
10549    commands: &[String],
10550    cwd: &Path,
10551    timeout: Duration,
10552) -> (Vec<CommandOutcome>, Vec<u32>) {
10553    if commands.is_empty() {
10554        // Nothing to mark as running and nothing to clear — an empty list
10555        // means "not configured", and touching `active` (or the disk) over
10556        // that would be a write for every round of a repo with no
10557        // `verify.e2e` / `verify.gate` commands at all.
10558        return (Vec::new(), Vec::new());
10559    }
10560    let mut out = Vec::new();
10561    let mut timed_out_pids = Vec::new();
10562    let total = commands.len();
10563    for (idx, command) in commands.iter().enumerate() {
10564        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10565        if let Err(e) = state.save() {
10566            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10567        }
10568        let started = Instant::now();
10569        let mut cmd = tokio::process::Command::new(&shell[0]);
10570        cmd.quiet();
10571        cmd.args(&shell[1..])
10572            .arg(command)
10573            .current_dir(cwd)
10574            .stdin(std::process::Stdio::null())
10575            .stdout(std::process::Stdio::piped())
10576            .stderr(std::process::Stdio::piped())
10577            .kill_on_drop(true);
10578        let mut outputs = Vec::new();
10579        let protected = (|| -> Result<()> {
10580            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10581                outputs.push(guard);
10582            }
10583            if let Some(cache) = state.config.cache_dir() {
10584                outputs.push(crate::retention::OutputLease::cache(
10585                    &crate::run::home(),
10586                    &cache,
10587                    false,
10588                )?);
10589            }
10590            Ok(())
10591        })();
10592        if let Err(e) = protected {
10593            out.push(CommandOutcome {
10594                command: command.clone(),
10595                code: None,
10596                output_tail: format!("output ownership unavailable: {e:#}"),
10597                duration_ms: 0,
10598                resource_blocked: true,
10599            });
10600            continue;
10601        }
10602        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10603        let (code, body) = match spawned {
10604            Ok(mut running) => {
10605                let pid = running.child().id();
10606                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10607                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10608                let mut paths = vec![cwd.to_owned()];
10609                if let Some(cache) = state.config.cache_dir() {
10610                    paths.push(cache);
10611                }
10612                let mut pressure = None;
10613                let status = tokio::select! {
10614                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10615                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10616                        pressure = Some(reason); None
10617                    }
10618                };
10619                let (code, failure) = match status {
10620                    Some(Ok(Ok(status))) => (
10621                        status.code(),
10622                        status
10623                            .code()
10624                            .is_none()
10625                            .then(|| format!("command terminated: {status}")),
10626                    ),
10627                    Some(Ok(Err(error))) => {
10628                        (None, Some(format!("failed waiting for command: {error}")))
10629                    }
10630                    Some(Err(_)) => {
10631                        // Leased cleanup confirms process-tree death itself.
10632                        // Unleased commands still need the caller's PID check
10633                        // after kill-on-drop, before releasing cache ownership.
10634                        if !running.coordinated()
10635                            && let Some(pid) = pid
10636                        {
10637                            timed_out_pids.push(pid);
10638                        }
10639                        (
10640                            None,
10641                            Some(format!("timed out after {}s", timeout.as_secs())),
10642                        )
10643                    }
10644                    None => (None, None),
10645                };
10646                running.finish().await;
10647                if let Some(reason) = pressure {
10648                    state.resource_block = Some(crate::run::ResourceBlock {
10649                        node: node.to_owned(),
10650                        kind: "disk pressure".to_owned(),
10651                        detail: reason.clone(),
10652                        at: Timestamp::now(),
10653                    });
10654                    out.push(CommandOutcome {
10655                        command: command.clone(),
10656                        code: None,
10657                        output_tail: reason,
10658                        duration_ms: started.elapsed().as_millis() as u64,
10659                        resource_blocked: true,
10660                    });
10661                    break;
10662                }
10663                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10664                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10665                if let Some(failure) = failure {
10666                    body.push('\n');
10667                    body.push_str(&failure);
10668                }
10669                (code, body)
10670            }
10671            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10672        };
10673        out.push(CommandOutcome {
10674            command: command.clone(),
10675            code,
10676            output_tail: tail(&body, OUTPUT_TAIL),
10677            duration_ms: started.elapsed().as_millis() as u64,
10678            resource_blocked: false,
10679        });
10680    }
10681    state.task_finished(task);
10682    if let Err(e) = state.save() {
10683        tracing::warn!("could not persist the end of {task}: {e:#}");
10684    }
10685    (out, timed_out_pids)
10686}
10687
10688/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10689/// section (`report::run`) and in the `merge` event this node records — for
10690/// the operator to run by hand.
10691///
10692/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10693/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10694/// merge commits", or "require linear history") rejects the push a `--no-ff`
10695/// merge would produce, which is exactly the guidance this function replaces.
10696/// `message`'s first line becomes the squash commit's subject, matching the
10697/// note `report::run` prints alongside this command — see that function for
10698/// why an explicit subject is not optional there.
10699fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10700    let repo = repo.display();
10701    match style {
10702        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10703        MergeStyle::Squash => {
10704            // The subject sits inside double quotes, and a title an agent
10705            // wrote may carry the characters that break out of them.
10706            let subject = message
10707                .lines()
10708                .next()
10709                .unwrap_or(branch)
10710                .replace(['\\', '"', '$', '`'], "");
10711            format!(
10712                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10713            )
10714        }
10715        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10716    }
10717}
10718
10719/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10720/// under the hood, rejects a `title` over 256 characters and the whole
10721/// command fails — no PR at all, for a run whose body was otherwise fine
10722/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10723/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10724/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10725/// this bound. It is a margin, not a guarantee — a title packed
10726/// with multi-unit characters could still in principle land close to the
10727/// edge, but a real task title's occasional emoji or accented letter fits
10728/// comfortably inside it.
10729const PR_TITLE_MAX: usize = 240;
10730
10731/// A pull request title from the opening line of `text`, or `None` when that
10732/// line is not English (GitHub text is) or has no letters.
10733///
10734/// A line within `max` is kept as is. A longer one is cut at the end of its
10735/// first sentence when that falls inside `max`, else at a word boundary with a
10736/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10737/// make every merely-truncated title look non-English). The language check
10738/// runs on the kept text before any mark is added, so only what GitHub will
10739/// show is judged: an English opening followed by non-ASCII far past the cut
10740/// still passes.
10741fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10742    let line = queue::first_line(text)?;
10743    let chars: Vec<char> = line.chars().collect();
10744    let (kept, mark) = if chars.len() <= max {
10745        (line.to_owned(), "")
10746    } else if let Some(end) = sentence_end(&chars, max) {
10747        (chars[..end].iter().collect::<String>(), "")
10748    } else {
10749        let room = max.saturating_sub(3);
10750        // Cut at the last space inside the room; when the char just past the
10751        // room is a space the room already ends on a word.
10752        let cut = if chars[room].is_whitespace() {
10753            room
10754        } else {
10755            chars[..room]
10756                .iter()
10757                .rposition(|c| c.is_whitespace())
10758                .unwrap_or(room)
10759        };
10760        let head: String = chars[..cut].iter().collect();
10761        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10762        (head.to_owned(), "...")
10763    };
10764    if kept.is_empty() {
10765        return None;
10766    }
10767    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10768        return None;
10769    }
10770    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10771        return None;
10772    }
10773    Some(format!("{kept}{mark}"))
10774}
10775
10776/// The char length of the first sentence of `chars` when it ends within `max`
10777/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10778/// abbreviations so `e.g. foo` does not end a title early.
10779fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10780    const MIN: usize = 20;
10781    for i in MIN..max.min(chars.len()) {
10782        if !matches!(chars[i], '.' | '!' | '?') {
10783            continue;
10784        }
10785        let Some(&next) = chars.get(i + 1) else {
10786            continue;
10787        };
10788        if !next.is_whitespace() {
10789            continue;
10790        }
10791        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10792        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10793            continue;
10794        }
10795        let word: String = chars[..i]
10796            .iter()
10797            .rev()
10798            .take_while(|c| !c.is_whitespace())
10799            .collect::<Vec<_>>()
10800            .into_iter()
10801            .rev()
10802            .collect();
10803        let word = word.to_ascii_lowercase();
10804        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10805            continue;
10806        }
10807        let end = chars[..i]
10808            .iter()
10809            .rposition(|c| !c.is_whitespace())
10810            .map_or(i, |p| p + 1);
10811        return Some(end);
10812    }
10813    None
10814}
10815
10816/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10817/// change: a title and a body describing what was *implemented*, not the task
10818/// that asked for it. A task reads as a request; a reader of the merged
10819/// history wants the change.
10820struct PrMessage {
10821    title: String,
10822    body: String,
10823}
10824
10825impl PrMessage {
10826    /// Title, blank line, body. The first line is the squash/merge commit
10827    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10828    /// has to stay one sensible line.
10829    fn commit_message(&self) -> String {
10830        format!("{}\n\n{}", self.title, self.body)
10831    }
10832}
10833
10834/// The text after a leading `TITLE:` (any case) on `line`.
10835fn title_marker(line: &str) -> Option<&str> {
10836    let line = line.trim();
10837    let head = line.get(..6)?;
10838    head.eq_ignore_ascii_case("title:")
10839        .then(|| line[6..].trim())
10840}
10841
10842/// The implementer's own one-line title: the `TITLE:` line the implement
10843/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10844/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10845/// source, and a title that says as much is refused here too.
10846fn summary_title(summary: &str) -> Option<String> {
10847    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10848    let raw = title_marker(first)?;
10849    if raw.is_empty() {
10850        return None;
10851    }
10852    let title = queue::title_from(raw, PR_TITLE_MAX);
10853    let lower = title.to_ascii_lowercase();
10854    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10855        return None;
10856    }
10857    Some(title)
10858}
10859
10860/// How `open_review`'s instruction begins; see [`landing_title`].
10861const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10862
10863/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10864pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10865
10866/// Does this run review work that already existed, rather than implement a
10867/// task? Runs recorded before `reviewed_commits` existed carry only the
10868/// candidate marker.
10869fn is_review_run(state: &RunState) -> bool {
10870    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10871}
10872
10873/// The title of a review-only run: the subject of the oldest commit under
10874/// review. Later commits are usually fixups, and `instruction` is the review
10875/// prompt, which says nothing about the change. GitHub text is English, so a
10876/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10877fn review_title(state: &RunState) -> Option<String> {
10878    english_subject(
10879        state.reviewed_commits.as_ref()?.first()?,
10880        state.config.graph.github_english(),
10881    )
10882}
10883
10884/// `raw` as a pull request title, or `None` when it is blank, not English
10885/// (GitHub text is), or one of magi's own candidate commit subjects.
10886fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10887    let raw = raw.trim();
10888    if raw.is_empty() {
10889        return None;
10890    }
10891    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10892        return None;
10893    }
10894    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10895        return None;
10896    }
10897    let title = queue::title_from(raw, PR_TITLE_MAX);
10898    let lower = title.to_ascii_lowercase();
10899    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10900        return None;
10901    }
10902    Some(title)
10903}
10904
10905/// What a review-only run's branch says about itself, read at the moment the
10906/// pull request is opened.
10907#[derive(Debug, Clone, PartialEq, Eq)]
10908struct BranchFacts {
10909    /// `(subject, body)` of each commit, oldest first.
10910    commits: Vec<(String, String)>,
10911    /// Trimmed `git diff --stat`.
10912    stat: String,
10913}
10914
10915/// Longest diff stat shown: this many file lines plus the summary line.
10916const STAT_FILE_LINES: usize = 25;
10917/// Cap on the commit list, well inside GitHub's 65536-character body limit.
10918const COMMITS_MAX_CHARS: usize = 20_000;
10919
10920/// Read the commits and diff stat of `base..branch`. `None` when git cannot
10921/// say or finds nothing, so the caller falls back to what the run recorded.
10922async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10923    let commits = git::commit_log(repo, base, branch).await.ok()?;
10924    if commits.is_empty() {
10925        return None;
10926    }
10927    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10928    let lines: Vec<&str> = stat.lines().collect();
10929    let stat = if lines.len() > STAT_FILE_LINES + 1 {
10930        let omitted = lines.len() - 1 - STAT_FILE_LINES;
10931        let more = format!(" ... {omitted} more file(s)");
10932        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10933        kept.push(&more);
10934        kept.push(lines[lines.len() - 1]);
10935        kept.join("\n")
10936    } else {
10937        lines.join("\n")
10938    };
10939    Some(BranchFacts { commits, stat })
10940}
10941
10942/// Defang what would break the surrounding markdown: a closing `</details>`
10943/// and a code fence.
10944fn markdown_safe(text: &str) -> String {
10945    text.replace("</details>", "&lt;/details&gt;")
10946        .replace("\x60\x60\x60", "~~~")
10947}
10948
10949fn neutral_title(state: &RunState, winner: char) -> String {
10950    format!(
10951        "chore: land candidate {} of run {}",
10952        winner.to_ascii_uppercase(),
10953        state.id
10954    )
10955}
10956
10957/// The pull request title to hand to `land::merge_subject`. A review-only run
10958/// opened by an earlier build titled its pull request with the review prompt;
10959/// that title is dropped (empty, so the fallback applies) rather than landed.
10960/// Any other title, including an operator's rename, passes through untouched,
10961/// and so does every title of a run that implements a task.
10962pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10963    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10964        ""
10965    } else {
10966        pr_title
10967    }
10968}
10969
10970/// What the squash subject falls back to when the pull request title is empty
10971/// or candidate-shaped: for a review-only run the derived title, never the
10972/// review prompt held in `instruction`.
10973pub fn landing_subject_source(state: &RunState) -> String {
10974    if is_review_run(state) {
10975        let winner = state.candidates.first().map_or('A', |c| c.label);
10976        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10977    }
10978    state.instruction.clone()
10979}
10980
10981/// `summary` without its `TITLE:` line, which the pull request title already
10982/// carries.
10983fn summary_without_title(summary: &str) -> String {
10984    let mut lines = summary.trim().lines().peekable();
10985    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10986        lines.next();
10987    }
10988    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10989}
10990
10991/// The pull request title and body for the winning candidate.
10992///
10993/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
10994/// the task's own opening line via [`queue::title_from`] when there is none.
10995/// `state.instruction` can open with blank lines (`task_text` only rejects a
10996/// body that is blank *entirely*), which `title_from` skips.
10997///
10998/// Publishing uses an independent description generated from the final diff
10999/// and structured evidence. Internal summaries and reports remain local.
11000/// Non-publishing previews may display the original task and candidate summary.
11001/// The footer retains the run and candidate provenance.
11002#[cfg(test)]
11003fn pr_message(state: &RunState, winner: char) -> PrMessage {
11004    pr_message_with(state, winner, None)
11005}
11006
11007/// [`pr_message`] with what the branch of a review-only run says about itself.
11008/// `facts` is ignored for a run that implements a task.
11009#[cfg(test)]
11010fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11011    let raw = pr_message_raw(
11012        state,
11013        winner,
11014        facts,
11015        crate::pr_outcome::FinalHead::Unreported,
11016        ChangeNote::NotAsked,
11017    );
11018    let id = crate::scrub::Identity::current();
11019    PrMessage {
11020        title: crate::scrub::scrub(&raw.title, &id),
11021        body: crate::scrub::scrub(&raw.body, &id),
11022    }
11023}
11024
11025/// What the public description may say about the change itself.
11026#[derive(Debug, Clone, Copy)]
11027enum ChangeNote<'a> {
11028    /// A local preview or non-publishing merge mode: no public generation needed.
11029    NotAsked,
11030    /// A summary written from the final diff.
11031    Summary(&'a crate::run::ChangeSummary),
11032    /// A summary was asked for and could not be produced; the text says why.
11033    Unavailable(&'a str),
11034}
11035
11036fn pr_message_raw(
11037    state: &RunState,
11038    winner: char,
11039    facts: Option<&BranchFacts>,
11040    head: crate::pr_outcome::FinalHead<'_>,
11041    change: ChangeNote<'_>,
11042) -> PrMessage {
11043    if !matches!(change, ChangeNote::NotAsked) {
11044        let (title, mut body) = match change {
11045            ChangeNote::Summary(c) => (
11046                c.title.clone(),
11047                format!("## Summary\n\n{}\n\n", c.body.trim()),
11048            ),
11049            ChangeNote::Unavailable(why) => (
11050                neutral_title(state, winner),
11051                format!(
11052                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11053                ),
11054            ),
11055            ChangeNote::NotAsked => unreachable!(),
11056        };
11057        body.push_str(&public_pr_evidence(state, head));
11058        body.push_str(&format!(
11059            "\n---\nmagi:run/{} magi:candidate-{}\n",
11060            state.id,
11061            winner.to_ascii_lowercase()
11062        ));
11063        return PrMessage { title, body };
11064    }
11065    pr_preview(state, winner, facts, head)
11066}
11067
11068/// Append public facts from the run records after every prose generation,
11069/// including posting-gate rewrites. The agent cannot drop this provenance.
11070fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11071    let mut body = crate::pr_outcome::review_rounds(state);
11072    body.push_str(&crate::pr_outcome::final_verification(state, head));
11073    let open = state.last_round_findings();
11074    if !open.is_empty() {
11075        body.push_str("## Open review findings\n\n");
11076        for f in open {
11077            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11078        }
11079        body.push('\n');
11080    }
11081    if let Some(c) = state.change_summary.as_ref()
11082        && head == crate::pr_outcome::FinalHead::At(&c.head)
11083    {
11084        body.push_str(&format!(
11085            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11086            c.base, c.head
11087        ));
11088    }
11089    body
11090}
11091
11092/// Local-only preview of the operator's task and candidate summary.
11093fn pr_preview(
11094    state: &RunState,
11095    winner: char,
11096    facts: Option<&BranchFacts>,
11097    head: crate::pr_outcome::FinalHead<'_>,
11098) -> PrMessage {
11099    let summary = state
11100        .candidates
11101        .iter()
11102        .find(|c| c.label == winner)
11103        .map(|c| c.summary.as_str())
11104        .unwrap_or_default();
11105    // The fallback is the operator's own words and may not be English; GitHub
11106    // text always is, so a non-English task gets a neutral title instead.
11107    let review = is_review_run(state);
11108    let title = if review {
11109        facts
11110            .and_then(|f| {
11111                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11112            })
11113            .or_else(|| review_title(state))
11114            .or_else(|| {
11115                state
11116                    .candidates
11117                    .iter()
11118                    .find(|c| c.label == winner)
11119                    .filter(|c| !c.branch.starts_with("magi/"))
11120                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11121            })
11122            .unwrap_or_else(|| neutral_title(state, winner))
11123    } else {
11124        summary_title(summary).unwrap_or_else(|| {
11125            english_title(
11126                &state.instruction,
11127                PR_TITLE_MAX,
11128                state.config.graph.github_english(),
11129            )
11130            .unwrap_or_else(|| neutral_title(state, winner))
11131        })
11132    };
11133
11134    let mut body = String::new();
11135    let what = summary_without_title(summary);
11136    if !what.is_empty() {
11137        body.push_str("## Summary\n\n");
11138        body.push_str(&what);
11139        body.push_str("\n\n");
11140    }
11141
11142    // Review history uses structured facts across all rounds, never reports.
11143    body.push_str(&crate::pr_outcome::review_rounds(state));
11144    body.push_str(&crate::pr_outcome::final_verification(state, head));
11145    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11146
11147    let open = state.open_findings();
11148    if !open.is_empty() {
11149        body.push_str("## Open review findings\n\n");
11150        for f in &open {
11151            body.push_str(&format!(
11152                "- `{}` [{:?}] {}\n",
11153                f.id, f.severity, "Context retained in local review records."
11154            ));
11155        }
11156        body.push('\n');
11157    }
11158
11159    if let Some(fix) = fix
11160        && !fix.rejected.is_empty()
11161    {
11162        body.push_str("## Declined by the fixer\n\n");
11163        for r in &fix.rejected {
11164            body.push_str(&format!(
11165                "- `{}`: {}\n",
11166                r.id, "Reason retained in local review records."
11167            ));
11168        }
11169        body.push('\n');
11170    }
11171
11172    if review {
11173        // The review prompt is not the task; list what the branch carries.
11174        body.push_str("## Commits under review\n\n");
11175        if let Some(facts) = facts {
11176            let mut left = COMMITS_MAX_CHARS;
11177            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11178                let mut entry = format!("- {}\n", markdown_safe(subject));
11179                for l in markdown_safe(text).lines() {
11180                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11181                }
11182                if left == 0 {
11183                    body.push_str(&format!(
11184                        "- ... {} more commit(s)\n",
11185                        facts.commits.len() - i
11186                    ));
11187                    break;
11188                }
11189                if entry.len() > left {
11190                    // Even the first commit is cut: one huge body must not
11191                    // push the whole description past GitHub's limit.
11192                    let mut end = left;
11193                    while !entry.is_char_boundary(end) {
11194                        end -= 1;
11195                    }
11196                    entry.truncate(end);
11197                    entry.push_str("\n  ... (truncated)\n");
11198                    left = 0;
11199                } else {
11200                    left -= entry.len();
11201                }
11202                body.push_str(&entry);
11203            }
11204            if !facts.stat.trim().is_empty() {
11205                body.push_str(&format!(
11206                    "\n## Diff stat\n\n```\n{}\n```\n",
11207                    markdown_safe(facts.stat.trim())
11208                ));
11209            }
11210        } else {
11211            match &state.reviewed_commits {
11212                Some(subjects) => {
11213                    for s in subjects {
11214                        body.push_str(&format!("- {}\n", s.trim()));
11215                    }
11216                }
11217                None => {
11218                    // An older run kept only the prompt, with the commit list
11219                    // after its first paragraph.
11220                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11221                    body.push_str(rest.trim());
11222                    body.push('\n');
11223                }
11224            }
11225        }
11226    } else {
11227        let task = state.instruction.trim();
11228        let task = if task.is_empty() {
11229            "(empty task)"
11230        } else {
11231            task
11232        };
11233        body.push_str(&format!(
11234            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11235            task.replace("</details>", "&lt;/details&gt;")
11236        ));
11237    }
11238
11239    body.push_str(&format!(
11240        "\n---\nmagi:run/{} magi:candidate-{}\n",
11241        state.id,
11242        winner.to_ascii_lowercase()
11243    ));
11244
11245    PrMessage { title, body }
11246}
11247
11248/// The task with what the repository says about the existing work it names
11249/// appended, so an implementer knows what it started from and what it must
11250/// not redo. Unchanged when the task names nothing.
11251fn seeded_instruction(state: &RunState) -> String {
11252    match refs::describe(&state.seeds) {
11253        Some(facts) => format!(
11254            "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11255             Candidates start from the unmerged branch named above, when there \
11256             is one, and carry any unmerged commit named by sha as a \
11257             cherry-pick. Check that this is what the task meant before \
11258             building on it.",
11259            state.instruction
11260        ),
11261        None => state.instruction.clone(),
11262    }
11263}
11264
11265/// Does the winner have no commits ahead of the base it would land on?
11266/// Any failure to find out reads as "not empty": the merge then behaves as it
11267/// always did rather than refusing on a guess.
11268async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11269    let base = &state.base_branch;
11270    let mut against = base.clone();
11271    if mode == MergeMode::Pr {
11272        // A pull request lands on the remote's base, never the local branch
11273        // of the same name: if that cannot be read, "not empty" is the safe
11274        // answer.
11275        let remote = &state.config.merge.remote;
11276        let tracking = format!("{remote}/{base}");
11277        let fetched = git::fetch(repo, remote, base).await;
11278        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11279            against = tracking;
11280        } else {
11281            return false;
11282        }
11283    }
11284    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11285}
11286
11287/// Why nothing was opened for an empty winner, with what the task's own
11288/// references resolved to.
11289fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11290    let mut detail = format!(
11291        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11292         nothing to open a pull request for"
11293    );
11294    match refs::describe(&state.seeds) {
11295        Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11296        None => detail.push_str(
11297            "\nThe task names no existing branch or commit; if it means to land work \
11298             that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11299        ),
11300    }
11301    detail
11302}
11303
11304/// What the `Pr` merge does once it knows whether the branch already has an
11305/// open pull request.
11306#[derive(Debug, PartialEq, Eq)]
11307enum PrPlan {
11308    Create,
11309    Adopt { url: String, title: String },
11310    Stop(String),
11311}
11312
11313/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11314/// a failed lookup -> stop with the real reason. Never guesses.
11315fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11316    match found {
11317        Ok(land::OpenPr::None) => PrPlan::Create,
11318        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11319        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11320            "several open pull requests exist for this branch, not picking one: {}",
11321            urls.join(" ")
11322        )),
11323        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11324    }
11325}
11326
11327/// `gh pr create`, returning the PR url.
11328pub(crate) async fn gh_pr_create(
11329    cwd: &Path,
11330    base: &str,
11331    head: &str,
11332    title: &str,
11333    body: &str,
11334) -> Result<String> {
11335    let out = tokio::process::Command::new("gh")
11336        .args([
11337            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11338        ])
11339        .current_dir(cwd)
11340        .quiet()
11341        .stdin(std::process::Stdio::null())
11342        .output()
11343        .await
11344        .context("spawn gh")?;
11345    if out.status.success() {
11346        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11347    } else {
11348        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11349    }
11350}
11351
11352/// Never force deletion of uncommitted work, including unknown legacy trees.
11353async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11354    match std::fs::symlink_metadata(path) {
11355        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11356            // A previous cleanup may have removed the tree before a branch
11357            // deletion or state save failed. Unregister it if still present.
11358            git::worktree_remove_clean(repo, path).await?;
11359            return Ok(());
11360        }
11361        Err(e) => return Err(e.into()),
11362        Ok(_) => {}
11363    }
11364    if !git::is_clean(path).await? {
11365        bail!(
11366            "worktree has uncommitted work; refusing removal: {}",
11367            path.display()
11368        );
11369    }
11370    if !git::worktree_remove_clean(repo, path).await? {
11371        bail!("worktree removal refused: {}", path.display());
11372    }
11373    Ok(())
11374}
11375
11376/// Tear a run's worktrees and branches down.
11377///
11378/// `home` is where the updated `run.json` is saved (via
11379/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11380/// a housekeeping pass already has its own honest `home` handed to it, and
11381/// falling through to the global here would write back through whichever
11382/// directory some other process or test pinned into that `OnceLock` first,
11383/// not the one the caller actually resolved its `runs` and `state` from.
11384pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11385    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11386        bail!("refusing fold: {reason}");
11387    }
11388    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11389        bail!("candidate preservation incomplete; refusing cleanup");
11390    }
11391    // Serialize ordinary folding with the same starts/resumes as target-only
11392    // eviction. Otherwise a second janitor could remove the whole worktree
11393    // around a live output borrower even though target eviction was fenced.
11394    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11395    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11396    let _driver = crate::driver::maintenance(home, &state.id)?;
11397    let mut output_guards = Vec::new();
11398    for entry in std::fs::read_dir(state.worktree_root())
11399        .into_iter()
11400        .flatten()
11401        .flatten()
11402    {
11403        if entry.path().join(".git").is_file() {
11404            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11405        }
11406    }
11407    let repo = state.repo.clone();
11408    // CLI, Web and automatic cleanup all enter here, independently of
11409    // Runner::fold_losers. Protect deletion and shared config release too.
11410    let wait = repository_wait(state, None);
11411    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11412        Ok(lease) => lease,
11413        Err(e) => {
11414            state.save_under(home)?;
11415            return Err(e);
11416        }
11417    };
11418    // A merged PR proves only its published change. Preflight every source
11419    // before removing anything; a losing candidate or detached reviewer can
11420    // still retain independent commits or Git recovery checkpoints.
11421    let _source_outputs = if state.status == RunStatus::Merged
11422        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11423    {
11424        verify_merged_fold_sources(state, drop_winner, home).await?
11425    } else {
11426        Vec::new()
11427    };
11428    let root = state.worktree_root();
11429    let winner = state.tally.as_ref().map(|t| t.winner);
11430    let mut removed = Vec::new();
11431
11432    for i in 0..state.candidates.len() {
11433        let c = state.candidates[i].clone();
11434        let is_winner = Some(c.label) == winner;
11435        if is_winner && !drop_winner {
11436            continue;
11437        }
11438        let existed = c.worktree.exists();
11439        // A vanished directory can still have a Git registration that keeps
11440        // its branch checked out. Remove that registration before the branch.
11441        remove_preserved_worktree(&repo, &c.worktree).await?;
11442        if existed {
11443            removed.push(c.worktree.to_string_lossy().into_owned());
11444        }
11445        // A branch handed to a later run (and its pull request) is not this
11446        // run's to delete.
11447        let handed_over = state.released_branches.contains(&c.branch);
11448        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11449            if !git::branch_delete(&repo, &c.branch).await? {
11450                bail!("candidate branch removal refused");
11451            }
11452            removed.push(c.branch.clone());
11453        }
11454        state.candidates[i].folded = true;
11455    }
11456
11457    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11458        let path = name.path();
11459        let keep = !drop_winner
11460            && winner.is_some_and(|w| {
11461                path.file_name()
11462                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11463            });
11464        if keep {
11465            continue;
11466        }
11467        remove_preserved_worktree(&repo, &path).await?;
11468        removed.push(path.to_string_lossy().into_owned());
11469    }
11470
11471    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11472    // judge worktrees, so once the loop above has cleared all of them out,
11473    // the parent is a bare directory nobody else was ever going to remove -
11474    // git only ever managed what was inside it. Left alone, one of these
11475    // accumulates per fully-folded run; the operator's own machine had 74.
11476    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11477    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11478    // so does anything a run never claimed that happens to share the bay.
11479    remove_if_empty(&root);
11480
11481    if state.enabled_worktree_config && drop_winner {
11482        // A release, not a raw disable: some sibling run in this repository
11483        // may still hold its own reference (see `git::acquire_worktree_config`),
11484        // and only the last release actually turns the setting back off.
11485        git::release_worktree_config(&repo).await.ok();
11486        state.enabled_worktree_config = false;
11487    }
11488    state.save_under(home)?;
11489    Ok(removed)
11490}
11491
11492async fn verify_merged_fold_sources(
11493    state: &RunState,
11494    drop_winner: bool,
11495    home: &Path,
11496) -> Result<Vec<std::fs::File>> {
11497    let winner = state.tally.as_ref().map(|t| t.winner);
11498    let mut heads = BTreeSet::new();
11499    for candidate in &state.candidates {
11500        if Some(candidate.label) == winner && !drop_winner {
11501            continue;
11502        }
11503        match std::fs::symlink_metadata(&candidate.worktree) {
11504            Ok(_) => anyhow::ensure!(
11505                candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11506                "candidate recovery source is outside the inventoried run root: {}",
11507                candidate.worktree.display()
11508            ),
11509            Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11510            Err(error) => {
11511                return Err(error).context(format!(
11512                    "cannot inspect candidate recovery source: {}",
11513                    candidate.worktree.display()
11514                ));
11515            }
11516        }
11517        if git::branch_exists(&state.repo, &candidate.branch).await? {
11518            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11519            heads.extend(
11520                git::git(
11521                    &state.repo,
11522                    &["reflog", "show", "--format=%H", &candidate.branch],
11523                )
11524                .await?
11525                .lines()
11526                .map(str::to_owned),
11527            );
11528        }
11529    }
11530    let mut cargo_guards = Vec::new();
11531    let entries = match std::fs::read_dir(state.worktree_root()) {
11532        Ok(entries) => Some(entries),
11533        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11534        Err(error) => return Err(error.into()),
11535    };
11536    for entry in entries.into_iter().flatten() {
11537        let path = entry?.path();
11538        let metadata = std::fs::symlink_metadata(&path)?;
11539        anyhow::ensure!(
11540            metadata.is_dir() && !metadata.file_type().is_symlink(),
11541            "non-directory or symlink recovery source retained: {}",
11542            path.display()
11543        );
11544        if !drop_winner
11545            && winner.is_some_and(|w| {
11546                path.file_name()
11547                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11548            })
11549        {
11550            continue;
11551        }
11552        anyhow::ensure!(
11553            path.join(".git").is_file(),
11554            "unregistered recovery source retained: {}",
11555            path.display()
11556        );
11557        anyhow::ensure!(
11558            git::git(
11559                &path,
11560                &[
11561                    "status",
11562                    "--porcelain",
11563                    "--untracked-files=all",
11564                    "--ignored",
11565                    "--",
11566                    ".",
11567                    ":(exclude)target"
11568                ]
11569            )
11570            .await?
11571            .is_empty(),
11572            "source changes or ignored recovery files retained: {}",
11573            path.display()
11574        );
11575        match std::fs::symlink_metadata(path.join("target")) {
11576            Ok(_) => {
11577                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11578            }
11579            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11580            Err(error) => return Err(error.into()),
11581        }
11582        for checkpoint in [
11583            "rebase-merge",
11584            "rebase-apply",
11585            "sequencer",
11586            "CHERRY_PICK_HEAD",
11587            "MERGE_HEAD",
11588            "REBASE_HEAD",
11589            "REVERT_HEAD",
11590            "BISECT_LOG",
11591            "AUTO_MERGE",
11592            "MERGE_MSG",
11593        ] {
11594            let location = git::git(
11595                &path,
11596                &[
11597                    "rev-parse",
11598                    "--path-format=absolute",
11599                    "--git-path",
11600                    checkpoint,
11601                ],
11602            )
11603            .await?;
11604            anyhow::ensure!(
11605                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11606                if e.kind() == std::io::ErrorKind::NotFound),
11607                "Git recovery checkpoint retained: {checkpoint}"
11608            );
11609        }
11610        heads.insert(git::rev_parse(&path, "HEAD").await?);
11611        heads.extend(
11612            git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11613                .await?
11614                .lines()
11615                .map(str::to_owned),
11616        );
11617        if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11618            heads.insert(original);
11619        }
11620    }
11621    if !heads.is_empty() {
11622        // Never trust a run-wide merge label or the local checkout's branch.
11623        // Pin the fetched base once. A squash does not preserve commit IDs:
11624        // prove the winner's cumulative patch before accepting its ancestors.
11625        let reference = format!(
11626            "refs/remotes/{}/{}",
11627            state.config.merge.remote, state.base_branch
11628        );
11629        let base = git::rev_parse(&state.repo, &reference)
11630            .await
11631            .context("cannot prove merged sources against the fetched base")?;
11632        let mut squash_winner = None;
11633        let mut checked_squash = false;
11634        for head in heads {
11635            if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11636                .await?
11637                .ok()
11638            {
11639                continue;
11640            }
11641            if !checked_squash {
11642                squash_winner = proven_squash_winner(state, &base).await?;
11643                checked_squash = true;
11644            }
11645            let incorporated = if let Some(winner) = &squash_winner {
11646                git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11647                    .await?
11648                    .ok()
11649            } else {
11650                false
11651            };
11652            anyhow::ensure!(
11653                incorporated,
11654                "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11655            );
11656        }
11657    }
11658    Ok(cargo_guards)
11659}
11660
11661/// Called only for a confirmed merged run. Its label admits proof, but does
11662/// not prove that any particular local branch or recovery checkpoint landed.
11663/// Match the complete binary patch, including modes and blob IDs, against a
11664/// single-parent commit in the pinned fetched history. This works after later
11665/// base commits too, without trusting the current base tree to still match.
11666async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11667    let Some(winner) = state.winner() else {
11668        return Ok(None);
11669    };
11670    let head = git::rev_parse(&state.repo, &winner.branch).await?;
11671    if !git::git_raw(
11672        &state.repo,
11673        &["merge-base", "--is-ancestor", &state.base_commit, &head],
11674    )
11675    .await?
11676    .ok()
11677    {
11678        return Ok(None);
11679    }
11680    let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11681    if patch.is_empty() {
11682        return Ok(None);
11683    }
11684    let range = format!("{}..{base}", state.base_commit);
11685    let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11686    for commit in commits.lines() {
11687        let parent = format!("{commit}^");
11688        if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11689            return Ok(Some(head));
11690        }
11691    }
11692    Ok(None)
11693}
11694
11695async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11696    git::git(
11697        repo,
11698        &[
11699            "-c",
11700            "core.quotePath=true",
11701            "diff",
11702            "--binary",
11703            "--full-index",
11704            "--no-renames",
11705            "--no-ext-diff",
11706            "--no-textconv",
11707            "--no-color",
11708            "--ignore-submodules=none",
11709            "--submodule=short",
11710            from,
11711            to,
11712            "--",
11713        ],
11714    )
11715    .await
11716}
11717
11718/// Remove `dir` if it exists and has nothing in it.
11719///
11720/// Best-effort and silent by design: a directory that is not empty (a run
11721/// whose winner is still parked there, a stray file some other process left)
11722/// is exactly the case this must refuse, and a directory that is already gone
11723/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11724/// already refuses a non-empty directory, so the emptiness check below is
11725/// belt, not suspenders - it is what keeps this from ever attempting the
11726/// removal in the case that matters, rather than trusting `remove_dir`'s
11727/// error path to have no side effects if it ever changed.
11728fn remove_if_empty(dir: &Path) {
11729    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11730        std::fs::remove_dir(dir).ok();
11731    }
11732}
11733
11734/// Severity of the worst open finding in the last review round, for reporting.
11735pub fn worst_open(state: &RunState) -> Option<Severity> {
11736    state
11737        .reviews
11738        .last()?
11739        .reviews
11740        .iter()
11741        .flat_map(|r| r.findings.iter())
11742        .map(|f| f.severity)
11743        .max()
11744}
11745
11746/// `[worktree] setup` hides its products per worktree, which needs
11747/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11748/// worktrees, like the hook's; every place that runs setup calls this first.
11749async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11750    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11751        git::acquire_worktree_config(repo).await?;
11752        state.enabled_worktree_config = true;
11753    }
11754    Ok(())
11755}
11756
11757#[cfg(test)]
11758mod tests {
11759    #[tokio::test]
11760    async fn launcher_base_fetch_requires_remote_success() {
11761        let tmp = tempfile::tempdir().unwrap();
11762        let repo = tmp.path().join("repo");
11763        let remote = tmp.path().join("remote");
11764        std::fs::create_dir(&repo).unwrap();
11765        std::fs::create_dir(&remote).unwrap();
11766        git::git(&remote, &["init", "--bare", "-b", "main"])
11767            .await
11768            .unwrap();
11769        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11770        git::git(
11771            &repo,
11772            &[
11773                "-c",
11774                "user.name=Fixture",
11775                "-c",
11776                "user.email=fixture@example.invalid",
11777                "commit",
11778                "--allow-empty",
11779                "-m",
11780                "fixture",
11781            ],
11782        )
11783        .await
11784        .unwrap();
11785        git::git(
11786            &repo,
11787            &["remote", "add", "origin", remote.to_str().unwrap()],
11788        )
11789        .await
11790        .unwrap();
11791        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11792        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11793        assert_eq!(
11794            resolve_base(&repo, "main", "origin", None)
11795                .await
11796                .unwrap()
11797                .commit,
11798            head
11799        );
11800        #[cfg(unix)]
11801        {
11802            let policy: crate::config::Coordination = toml::from_str(
11803                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11804            ).unwrap();
11805            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11806            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11807            let held = crate::repo_lock::SeatLease::operation_with(
11808                &repo,
11809                Some(&policy),
11810                &crate::repo_lock::WaitPolicy::default(),
11811            )
11812            .await
11813            .unwrap();
11814            assert!(held.is_some());
11815            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11816            tokio::pin!(fetch);
11817            assert!(
11818                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11819                    .await
11820                    .is_err()
11821            );
11822            assert!(
11823                !repo.join(".git/FETCH_HEAD").exists(),
11824                "no fetch before admission"
11825            );
11826            drop(held);
11827            let fetched = fetch.await.unwrap();
11828            assert_eq!(fetched.commit, head);
11829            assert_eq!(fetched.remote, "origin");
11830            assert_eq!(fetched.branch, "main");
11831        }
11832        // A surviving tracking ref is not successful fetch evidence.
11833        git::git(
11834            &repo,
11835            &[
11836                "remote",
11837                "set-url",
11838                "origin",
11839                tmp.path().join("missing").to_str().unwrap(),
11840            ],
11841        )
11842        .await
11843        .unwrap();
11844        let error = resolve_base(&repo, "main", "origin", None)
11845            .await
11846            .unwrap_err()
11847            .to_string();
11848        assert!(error.contains("cannot read origin/main"));
11849        assert!(error.contains("refusing to branch off"));
11850    }
11851
11852    #[test]
11853    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11854        let leaked = vec!["chore(deps): update a crate".to_owned()];
11855        let own = "fix(daemon): apply a chosen action";
11856        assert!(should_retitle("", own, &leaked));
11857        assert!(should_retitle(
11858            &format!("{REVIEW_PROMPT_OPENING} `x`"),
11859            own,
11860            &leaked
11861        ));
11862        assert!(should_retitle(
11863            "chore: land candidate A of run 1",
11864            own,
11865            &leaked
11866        ));
11867        assert!(should_retitle(
11868            "magi: candidate A (uncommitted work)",
11869            own,
11870            &leaked
11871        ));
11872        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11873        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11874        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11875    }
11876
11877    #[test]
11878    fn pr_merge_plan_creates_adopts_or_stops() {
11879        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11880        assert_eq!(
11881            pr_merge_plan(Ok(land::OpenPr::One {
11882                url: "u".into(),
11883                title: "t".into()
11884            })),
11885            PrPlan::Adopt {
11886                url: "u".into(),
11887                title: "t".into()
11888            }
11889        );
11890        let PrPlan::Stop(many) =
11891            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11892        else {
11893            panic!("many must stop");
11894        };
11895        assert!(many.contains('a') && many.contains('b'));
11896        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11897            panic!("a failed lookup must stop");
11898        };
11899        assert!(err.contains("bad token"));
11900    }
11901
11902    use super::*;
11903    use crate::run::GateStatus;
11904    use std::collections::BTreeMap;
11905    use std::time::Duration;
11906
11907    fn conductor() -> AgentSpec {
11908        AgentSpec {
11909            id: "conductor".to_owned(),
11910            kind: crate::config::AgentKind::Command,
11911            model: None,
11912            command: vec!["true".to_owned()],
11913            extra_args: Vec::new(),
11914            env: BTreeMap::new(),
11915            prompt_delivery: None,
11916        }
11917    }
11918
11919    fn spec(id: &str) -> AgentSpec {
11920        AgentSpec {
11921            id: id.to_owned(),
11922            kind: crate::config::AgentKind::Command,
11923            model: None,
11924            command: vec!["true".to_owned()],
11925            extra_args: Vec::new(),
11926            env: BTreeMap::new(),
11927            prompt_delivery: None,
11928        }
11929    }
11930
11931    fn ids(xs: &[&str]) -> BTreeSet<String> {
11932        xs.iter().map(|s| (*s).to_owned()).collect()
11933    }
11934
11935    #[test]
11936    fn pick_successor_skips_an_agent_another_seat_holds() {
11937        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
11938        let roster = [spec("a"), spec("b"), spec("c")];
11939        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11940        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11941    }
11942
11943    #[test]
11944    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11945        // The other seat started on c but was handed to d; c is free again.
11946        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11947        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11948        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11949        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11950        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11951    }
11952
11953    #[test]
11954    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11955        let roster = [spec("a"), spec("b")];
11956        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11957        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11958        let carried = ids(&["b"]);
11959        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11960        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11961    }
11962
11963    #[test]
11964    fn pick_successor_returns_none_without_any_untried_successor() {
11965        let roster = [spec("a"), spec("b")];
11966        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11967        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11968        let carried = ids(&["a"]);
11969        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11970    }
11971
11972    #[test]
11973    fn pick_successor_rescue_avoids_another_seats_occupant() {
11974        // b is a carried failure and free; a is held by the other seat.
11975        let roster = [spec("a"), spec("b"), spec("c")];
11976        let carried = ids(&["b", "c"]);
11977        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11978        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11979    }
11980
11981    #[test]
11982    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11983        let roster = [spec("a"), spec("b"), spec("c")];
11984        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11985        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11986    }
11987
11988    #[test]
11989    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11990        let roster = [spec("a"), spec("b"), spec("c")];
11991        let failed = ids(&["a", "b", "c"]);
11992        // Rescue looks at the whole roster, once per id, then runs out.
11993        let mut tried = ids(&["b"]);
11994        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11995        assert_eq!(first.id, "a");
11996        tried.insert(first.id.clone());
11997        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11998        assert_eq!(second.id, "c");
11999        tried.insert(second.id.clone());
12000        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12001    }
12002
12003    #[test]
12004    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12005        let roster = [spec("a"), spec("b")];
12006        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12007        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12008    }
12009
12010    #[test]
12011    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12012        let roster = [spec("a"), spec("b"), spec("c")];
12013        let h = SeatHistory {
12014            failed: ids(&["a"]),
12015            last_ok: Some("b".to_owned()),
12016            last_fail: None,
12017        };
12018        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12019        // A last answerer that left the roster, or later failed, is ignored.
12020        let gone = SeatHistory {
12021            last_ok: Some("zzz".to_owned()),
12022            ..h.clone()
12023        };
12024        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12025        let failed = SeatHistory {
12026            failed: ids(&["a", "b"]),
12027            last_ok: Some("b".to_owned()),
12028            last_fail: None,
12029        };
12030        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12031    }
12032
12033    #[test]
12034    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12035        let first = SeatState::new("review-1", "alpha", 7);
12036        let next = handover_seat("review-1", "gamma", 7);
12037        assert_ne!(first.claude_session, next.claude_session);
12038    }
12039
12040    #[test]
12041    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12042        let mut state = state_with_summary("x", "y");
12043        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12044        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12045        assert_ne!(a.claude_session, b.claude_session);
12046    }
12047
12048    #[test]
12049    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12050        let roster = [spec("a"), spec("b")];
12051        let h = SeatHistory {
12052            failed: ids(&["a", "b"]),
12053            ..SeatHistory::default()
12054        };
12055        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12056        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12057        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12058    }
12059
12060    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12061    // fallback loop depends on to terminate: it must walk forward from the
12062    // seat's own position, never restart at the front of the roster, and it
12063    // must never hand back an id already tried, however many times that id
12064    // happens to appear.
12065
12066    #[test]
12067    fn failure_signature_ignores_numbers_and_paths() {
12068        assert_eq!(
12069            failure_signature("exited with Some(2) and no usable output"),
12070            failure_signature("exited with Some(137) and no usable output")
12071        );
12072        assert_eq!(
12073            failure_signature("process exited with no usable output (exit code 2)"),
12074            failure_signature("process exited with no usable output (exit code 137)")
12075        );
12076        for (old, new) in [
12077            (
12078                "exited with Some(2) and no usable output",
12079                "process exited with no usable output (exit code 2)",
12080            ),
12081            (
12082                "exited with None and no usable output",
12083                "process exited with no usable output (exit code unavailable)",
12084            ),
12085        ] {
12086            let previous = FailClass::Other(failure_signature(old));
12087            let current = FailClass::Other(failure_signature(new));
12088            assert_eq!(previous, current, "resumption retains the failure class");
12089            assert!(!should_hand_over(Some(&previous), &current));
12090        }
12091        assert_eq!(
12092            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12093            failure_signature("cannot open /var/x.txt: denied")
12094        );
12095        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12096    }
12097
12098    #[test]
12099    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12100        let other = FailClass::Other("x".into());
12101        assert!(should_hand_over(None, &FailClass::Quota));
12102        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12103        assert!(should_hand_over(
12104            Some(&FailClass::Timeout),
12105            &FailClass::Timeout
12106        ));
12107        assert!(should_hand_over(None, &other));
12108        assert!(!should_hand_over(Some(&other), &other));
12109        assert!(should_hand_over(
12110            Some(&other),
12111            &FailClass::Other("y".into())
12112        ));
12113        // A quota or timeout in between ends the run of identical failures.
12114        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12115        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12116    }
12117
12118    #[test]
12119    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12120        let a = SeatState::new("judge-1", "alpha", 7);
12121        let b = handover_seat("judge-1", "beta", 7);
12122        assert_ne!(a.claude_session, b.claude_session);
12123        assert_eq!(b.turns, 0);
12124    }
12125
12126    #[test]
12127    fn a_timeout_is_classified_apart_from_other_failures() {
12128        assert_eq!(
12129            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12130            Some(FailClass::Timeout)
12131        );
12132        assert!(matches!(
12133            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12134            Some(FailClass::Other(_))
12135        ));
12136    }
12137
12138    #[test]
12139    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12140        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12141        let tried = BTreeSet::from(["beta".to_owned()]);
12142        // beta sits at index 1; the next candidate is gamma, never alpha —
12143        // which is very likely a different candidate slot's own agent.
12144        let next = next_untried_in_roster(&roster, 1, &tried);
12145        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12146    }
12147
12148    #[test]
12149    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12150        let roster = vec![spec("alpha"), spec("beta")];
12151        let tried = BTreeSet::from(["beta".to_owned()]);
12152        // beta is the roster's last entry: nothing follows it, and alpha —
12153        // earlier in the roster, almost certainly a different candidate
12154        // slot's own agent — must not be reached by wrapping back to it.
12155        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12156    }
12157
12158    #[test]
12159    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12160        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12161        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12162        // beta (index 1) and gamma (index 2, the only entry after it) have
12163        // both been tried; alpha (index 0) never has, but it comes before
12164        // beta's own position, so there is nothing further for this seat.
12165        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12166    }
12167
12168    #[test]
12169    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12170        let roster = vec![spec("a"), spec("a"), spec("b")];
12171        let tried = BTreeSet::from(["a".to_owned()]);
12172        let next = next_untried_in_roster(&roster, 0, &tried);
12173        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12174    }
12175
12176    #[test]
12177    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12178        let roster = vec![spec("a"), spec("b")];
12179        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12180        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12181    }
12182
12183    #[test]
12184    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12185        let dir = tempfile::tempdir().unwrap();
12186        let bay = dir.path().join("ffff");
12187
12188        // Not there yet: nothing to do, nothing to panic on.
12189        remove_if_empty(&bay);
12190        assert!(!bay.exists());
12191
12192        // Something still inside - the winner's worktree, or a stray file -
12193        // keeps the directory standing.
12194        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12195        remove_if_empty(&bay);
12196        assert!(bay.exists(), "non-empty directory must survive");
12197
12198        // Once the last entry is gone, so is the directory itself.
12199        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12200        remove_if_empty(&bay);
12201        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12202    }
12203
12204    // `round_is_clean` is the exact decision this task fixed: a round with a
12205    // seat that never answered must not read the same as a round every seat
12206    // actually reviewed. These are deterministic and process-free by design —
12207    // the equivalent end-to-end check (a real reviewer timing out under a
12208    // live graph run) is a genuine race against wall-clock contention, and a
12209    // spawn slow enough to blow even a generous budget under a loaded test
12210    // run must not turn this specific regression check flaky.
12211
12212    #[test]
12213    fn a_full_panel_that_found_nothing_is_clean() {
12214        assert!(round_is_clean(
12215            0,
12216            true,
12217            2,
12218            2,
12219            0,
12220            IncompleteReviewPolicy::Block
12221        ));
12222    }
12223
12224    #[test]
12225    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12226        assert!(!round_is_clean(
12227            0,
12228            true,
12229            1,
12230            2,
12231            0,
12232            IncompleteReviewPolicy::Block
12233        ));
12234    }
12235
12236    #[test]
12237    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12238        assert!(!round_is_clean(
12239            1,
12240            true,
12241            1,
12242            2,
12243            0,
12244            IncompleteReviewPolicy::Warn
12245        ));
12246    }
12247
12248    #[test]
12249    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12250        assert!(round_is_clean(
12251            0,
12252            true,
12253            1,
12254            2,
12255            0,
12256            IncompleteReviewPolicy::Warn
12257        ));
12258    }
12259
12260    #[test]
12261    fn a_full_panel_with_an_open_finding_is_not_clean() {
12262        assert!(!round_is_clean(
12263            1,
12264            true,
12265            2,
12266            2,
12267            0,
12268            IncompleteReviewPolicy::Block
12269        ));
12270    }
12271
12272    #[test]
12273    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12274        assert!(!round_is_clean(
12275            0,
12276            false,
12277            2,
12278            2,
12279            0,
12280            IncompleteReviewPolicy::Block
12281        ));
12282    }
12283
12284    // The stall this task closes: under the default `block` policy, a seat
12285    // missing only because it was rate limited must not force a wait for a
12286    // session limit that will not lift by the next round. `round_is_clean`
12287    // is where that quorum carve-out lives; the review loop around it never
12288    // changes what a reviewer's vote or a finding's severity means.
12289
12290    #[test]
12291    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12292        // 1 of 2 answered, and the one missing was quota'd — the exact
12293        // "review-2 rate limited (quota)" shape from the field report.
12294        assert!(round_is_clean(
12295            0,
12296            true,
12297            1,
12298            2,
12299            1,
12300            IncompleteReviewPolicy::Block
12301        ));
12302    }
12303
12304    #[test]
12305    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12306        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12307        // not a quota loss (`quota_missing` stays 0) — worth another try.
12308        assert!(!round_is_clean(
12309            0,
12310            true,
12311            1,
12312            2,
12313            0,
12314            IncompleteReviewPolicy::Block
12315        ));
12316    }
12317
12318    #[test]
12319    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12320        assert!(!round_is_clean(
12321            1,
12322            true,
12323            1,
12324            2,
12325            1,
12326            IncompleteReviewPolicy::Block
12327        ));
12328        assert!(!round_is_clean(
12329            0,
12330            false,
12331            1,
12332            2,
12333            1,
12334            IncompleteReviewPolicy::Block
12335        ));
12336    }
12337
12338    #[test]
12339    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12340        // Every seat quota'd, nobody answered: there is no panel to decide
12341        // on, so this must fall through to the existing block-and-retry
12342        // fallback rather than call an unreviewed patch clean.
12343        assert!(!round_is_clean(
12344            0,
12345            true,
12346            0,
12347            2,
12348            2,
12349            IncompleteReviewPolicy::Block
12350        ));
12351    }
12352
12353    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12354        CommandOutcome {
12355            command: "test".to_owned(),
12356            code,
12357            output_tail: String::new(),
12358            duration_ms: 0,
12359            resource_blocked,
12360        }
12361    }
12362
12363    #[test]
12364    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12365        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12366        assert!(
12367            !verify_inconclusive(&[outcome(Some(1), false)]),
12368            "an ordinary failure is still evidence about the patch"
12369        );
12370        assert!(verify_inconclusive(&[outcome(None, true)]));
12371        assert!(
12372            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12373            "one inconclusive outcome taints the whole batch"
12374        );
12375        assert!(!verify_inconclusive(&[]));
12376    }
12377
12378    #[tokio::test]
12379    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12380        // Alive for the first two checks, then dead - confirms the loop
12381        // actually re-polls rather than deciding once and sleeping out the
12382        // ceiling regardless.
12383        let calls = std::sync::atomic::AtomicUsize::new(0);
12384        let started = Instant::now();
12385        wait_for_pids_with(
12386            &[123],
12387            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12388            Duration::from_millis(5),
12389            Duration::from_secs(5),
12390        )
12391        .await;
12392        assert!(
12393            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12394            "must keep checking rather than deciding on the first answer"
12395        );
12396        assert!(
12397            started.elapsed() < Duration::from_secs(1),
12398            "must return the moment it is confirmed dead, not wait out the ceiling"
12399        );
12400    }
12401
12402    #[tokio::test]
12403    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12404        let started = Instant::now();
12405        wait_for_pids_with(
12406            &[123],
12407            |_| true, // never reports dead
12408            Duration::from_millis(5),
12409            Duration::from_millis(30),
12410        )
12411        .await;
12412        let elapsed = started.elapsed();
12413        assert!(
12414            elapsed >= Duration::from_millis(30),
12415            "must not give up before its own ceiling: {elapsed:?}"
12416        );
12417        assert!(
12418            elapsed < Duration::from_secs(1),
12419            "must not wait past its own ceiling either: {elapsed:?}"
12420        );
12421    }
12422
12423    #[tokio::test]
12424    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12425        let started = Instant::now();
12426        wait_for_pids_with(
12427            &[],
12428            |_| true,
12429            Duration::from_secs(5),
12430            Duration::from_secs(5),
12431        )
12432        .await;
12433        assert!(
12434            started.elapsed() < Duration::from_millis(200),
12435            "an empty pid list has nothing to confirm"
12436        );
12437    }
12438
12439    // `review_conclusion` is the exact decision the review hand-off task
12440    // fixed: a round budget spent (or a tree that stopped moving) must not
12441    // collapse into `Blocked` regardless of what verification actually
12442    // said. Deterministic and process-free for the same reason the
12443    // `round_is_clean` family above is.
12444    fn review_round(
12445        clean: bool,
12446        blocking: usize,
12447        answered: usize,
12448        expected: usize,
12449        progressed: bool,
12450        e2e_ok: bool,
12451    ) -> ReviewRound {
12452        ReviewRound {
12453            round: 1,
12454            head: "h".to_owned(),
12455            verified_head: None,
12456            verified_at: None,
12457            reviews: Vec::new(),
12458            e2e: vec![CommandOutcome {
12459                command: "test".to_owned(),
12460                code: Some(if e2e_ok { 0 } else { 1 }),
12461                output_tail: String::new(),
12462                duration_ms: 0,
12463                resource_blocked: false,
12464            }],
12465            verify_retried: false,
12466            e2e_deferred: false,
12467            e2e_defer_reason: None,
12468            fix: None,
12469            blocking,
12470            answered,
12471            expected,
12472            clean,
12473            progressed,
12474            vote_split: false,
12475            reconsideration: Vec::new(),
12476            verdict: None,
12477        }
12478    }
12479
12480    #[test]
12481    fn review_conclusion_is_none_when_nothing_has_run() {
12482        assert_eq!(review_conclusion(&[], 3), None);
12483    }
12484
12485    #[test]
12486    fn review_conclusion_is_none_while_rounds_remain() {
12487        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12488        assert_eq!(review_conclusion(&rounds, 3), None);
12489    }
12490
12491    #[test]
12492    fn review_conclusion_is_gating_once_a_round_is_clean() {
12493        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12494        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12495    }
12496
12497    #[test]
12498    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12499        let rounds = vec![
12500            review_round(false, 1, 2, 2, true, true),
12501            review_round(false, 1, 2, 2, true, true),
12502        ];
12503        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12504    }
12505
12506    #[test]
12507    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12508        let rounds = vec![
12509            review_round(false, 1, 2, 2, true, true),
12510            review_round(false, 1, 2, 2, true, false),
12511        ];
12512        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12513    }
12514
12515    #[test]
12516    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12517        // Magi never got a command to run against this round's own head — a
12518        // resource-blocked attempt, not a red one — so this must never
12519        // settle on `Blocked` the way a genuine e2e failure would. `None`
12520        // here is what tells `Runner::review_loop` to retry the check
12521        // itself rather than trust this cheap recomputation with a verdict
12522        // it cannot actually produce.
12523        let mut blocked = review_round(false, 1, 2, 2, true, false);
12524        blocked.e2e[0].resource_blocked = true;
12525        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12526        assert_eq!(review_conclusion(&rounds, 2), None);
12527    }
12528
12529    #[test]
12530    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12531        // Missing input, not a verified tree — never a hand-off candidate.
12532        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12533        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12534    }
12535
12536    #[test]
12537    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12538        let rounds = vec![
12539            review_round(false, 1, 2, 2, false, true),
12540            review_round(false, 1, 2, 2, false, true),
12541        ];
12542        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12543    }
12544
12545    fn secs(n: u64) -> Duration {
12546        Duration::from_secs(n)
12547    }
12548
12549    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12550    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12551    #[tokio::test]
12552    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12553        ask_test_home();
12554        let tmp = tempfile::tempdir().unwrap();
12555        let repo = tmp.path().join("repo");
12556        std::fs::create_dir_all(&repo).unwrap();
12557        init_repo(&repo);
12558        let g = |args: &[&str]| {
12559            let out = std::process::Command::new("git")
12560                .args(args)
12561                .current_dir(&repo)
12562                .quiet()
12563                .output()
12564                .unwrap();
12565            assert!(
12566                out.status.success(),
12567                "{}",
12568                String::from_utf8_lossy(&out.stderr)
12569            );
12570        };
12571        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12572        g(&["checkout", "-b", "historical"]);
12573        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12574        g(&["add", "."]);
12575        g(&["commit", "-m", "old alternative"]);
12576        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12577        g(&["checkout", "-b", "magi/task-selected", &base]);
12578        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12579        g(&["add", "."]);
12580        g(&["commit", "-m", "measurement infrastructure"]);
12581        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12582        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12583        g(&["add", "."]);
12584        g(&["commit", "-m", "dependent implementation"]);
12585        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12586        g(&["checkout", "main"]);
12587        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12588        g(&["add", "."]);
12589        g(&["commit", "-m", "unrelated base change"]);
12590        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12591        let origin = tmp.path().join("origin.git");
12592        g(&[
12593            "clone",
12594            "--bare",
12595            repo.to_str().unwrap(),
12596            origin.to_str().unwrap(),
12597        ]);
12598        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12599        g(&["fetch", "origin"]);
12600        let selection = crate::pr_recovery::Selection {
12601            url: "https://example.test/project/pull/1".into(),
12602            branch: "magi/task-selected".into(),
12603            head: head.clone(),
12604            source_head: None,
12605            source_run: None,
12606            source_proof: None,
12607            disposition: Some(crate::pr_recovery::Disposition::Review),
12608            instruction: "preserve measured behavior".into(),
12609            approved_at: jiff::Timestamp::now(),
12610        };
12611        let mut template = runner_at(RunStatus::Reviewing);
12612        template.state.repo = repo.clone();
12613        template.state.base_commit = current.clone();
12614        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12615        template.state.origin = Some(crate::run::Origin::queue("selected"));
12616        template.state.pr_head = Some(selection.branch.clone());
12617        template.state.pr_recovery = Some(selection.clone());
12618        let mut runner =
12619            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12620                .await
12621                .unwrap();
12622        let wt = runner.state.winner().unwrap().worktree.clone();
12623        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12624        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12625        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12626        assert!(runner.state.seeds.is_empty());
12627        assert_eq!(runner.state.instruction, "preserve measured behavior");
12628        runner.sync_to_base().await.unwrap();
12629        assert_eq!(
12630            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12631            "prerequisite"
12632        );
12633        assert_eq!(
12634            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12635            "measured implementation"
12636        );
12637        assert_eq!(
12638            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12639            "current main change"
12640        );
12641        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12642        assert_eq!(
12643            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12644            head
12645        );
12646
12647        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12648        let mut partial = runner_at(RunStatus::Reviewing);
12649        partial.state.repo = repo.clone();
12650        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12651        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12652            run: Some("exit 1".into()),
12653            ..Default::default()
12654        }];
12655        let selection = runner.state.pr_recovery.clone().unwrap();
12656        partial.state.pr_recovery = Some(selection.clone());
12657        partial.state.pr_head = Some(selection.branch.clone());
12658        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12659        let id = partial.state.id.clone();
12660        let error = match Runner::open_selected(
12661            &repo,
12662            partial.state,
12663            partial.roles,
12664            1,
12665            runner.state.base_commit.clone(),
12666            selection,
12667        )
12668        .await
12669        {
12670            Ok(_) => panic!("setup must fail"),
12671            Err(e) => e,
12672        };
12673        assert_eq!(
12674            error
12675                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12676                .unwrap()
12677                .run,
12678            id
12679        );
12680        let failed = RunState::load(&id).unwrap();
12681        assert_eq!(failed.status, RunStatus::Blocked);
12682        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12683        assert!(Runner::resume(&id).is_err());
12684        runner.state.status = RunStatus::Ready;
12685        runner.state.driver_exited = true;
12686        runner.state.save().unwrap();
12687        let mut retry = runner_at(RunStatus::Reviewing);
12688        retry.state.repo = repo.clone();
12689        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12690        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12691        reselected.approved_at = jiff::Timestamp::now();
12692        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12693        retry.state.pr_head = Some(reselected.branch.clone());
12694        retry.state.pr_recovery = Some(reselected.clone());
12695        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12696            .await
12697            .unwrap();
12698        let restarted = Runner::open_selected(
12699            &repo,
12700            retry.state,
12701            retry.roles,
12702            1,
12703            runner.state.base_commit.clone(),
12704            reselected,
12705        )
12706        .await
12707        .unwrap();
12708        assert_eq!(
12709            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12710                .await
12711                .unwrap(),
12712            head
12713        );
12714        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12715        assert_eq!(
12716            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12717            head
12718        );
12719    }
12720
12721    #[tokio::test]
12722    #[cfg(unix)]
12723    async fn fallback_and_interrupted_restoration_require_coordination() {
12724        use std::os::fd::AsRawFd as _;
12725        ask_test_home();
12726        let tmp = tempfile::tempdir().unwrap();
12727        let mut runner = registered_runner(tmp.path()).await;
12728        runner.state.config.graph.timeout_resource_wait = 1;
12729        let winner = runner.state.candidates[0].clone();
12730        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12731        let leftover = winner.worktree.join("leftover");
12732        std::fs::write(&leftover, "untrusted edits").unwrap();
12733        let next = tmp.path().join("next-fixer");
12734        let mut failed = spec("first");
12735        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12736        let mut successor = spec("second");
12737        successor.command = vec![
12738            "sh".into(),
12739            "-c".into(),
12740            format!("touch '{}'; echo done", next.display()),
12741        ];
12742        let attempts = vec![
12743            (failed.clone(), "first".into()),
12744            (successor.clone(), "second".into()),
12745        ];
12746        let lock_path = runner.state.repo.join(".git/updater.lock");
12747        let external = std::fs::File::create(&lock_path).unwrap();
12748        assert_eq!(
12749            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12750            0
12751        );
12752        let artifacts = tmp.path().join("artifacts");
12753        let build = |spec, seat| SeatJob {
12754            spec,
12755            seat,
12756            cwd: winner.worktree.clone(),
12757            prompt: "fixture".into(),
12758            timeout: Duration::from_secs(10),
12759            allow_write: true,
12760            sessions: false,
12761            artifacts: artifacts.clone(),
12762            stem: "fix".into(),
12763            handover: None,
12764        };
12765        let (_, _, out) = runner
12766            .ask_fixer_among(
12767                attempts.clone(),
12768                "graft",
12769                None,
12770                Some((&winner.worktree, &head)),
12771                build,
12772            )
12773            .await;
12774        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12775        assert!(leftover.exists(), "contention must prevent clean");
12776        assert!(!next.exists());
12777        assert!(runner.state.handovers.is_empty());
12778        // Early interrupted-repair rollback uses the same protected route.
12779        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12780            from_run: "earlier".into(),
12781            generation: 1,
12782            mode: crate::recovery::Mode::VerifyRepair,
12783            reasons: Vec::new(),
12784            reviewed_head: None,
12785            rounds: Vec::new(),
12786            open_findings: Vec::new(),
12787            evidence: None,
12788            criteria_inherited: true,
12789            repair: crate::recovery::Repair::Started {
12790                before: head.clone(),
12791            },
12792        });
12793        assert!(
12794            runner
12795                .recovery_repair(&winner)
12796                .await
12797                .unwrap_err()
12798                .is::<crate::repo_lock::ResourceBlocked>()
12799        );
12800        assert!(matches!(
12801            runner.state.recovery.as_ref().unwrap().repair,
12802            crate::recovery::Repair::Started { .. }
12803        ));
12804        assert!(leftover.exists());
12805        drop(external);
12806        runner.recovery_repair(&winner).await.unwrap();
12807        assert!(!leftover.exists());
12808        assert!(matches!(
12809            runner.state.recovery.as_ref().unwrap().repair,
12810            crate::recovery::Repair::Done { .. }
12811        ));
12812        // A hard lock-open failure also leaves HEAD and untracked work intact.
12813        std::fs::remove_file(&lock_path).unwrap();
12814        std::fs::create_dir(&lock_path).unwrap();
12815        std::fs::write(&leftover, "untrusted edits").unwrap();
12816        let (_, _, out) = runner
12817            .ask_fixer_among(
12818                attempts,
12819                "graft",
12820                None,
12821                Some((&winner.worktree, &head)),
12822                build,
12823            )
12824            .await;
12825        assert!(matches!(out, AgentOutcome::Failed(_)));
12826        assert!(leftover.exists());
12827        assert!(!next.exists());
12828        assert_eq!(
12829            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12830            head
12831        );
12832    }
12833
12834    #[cfg(unix)]
12835    async fn registered_runner(root: &Path) -> Runner {
12836        ask_test_home();
12837        let repo = root.join("repo");
12838        std::fs::create_dir_all(&repo).unwrap();
12839        init_repo(&repo);
12840        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12841        let mut runner = runner_at(RunStatus::Prep);
12842        runner.state.repo = repo.clone();
12843        runner.state.config.coordination = crate::config::test_coordination();
12844        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12845        runner.state.config.graph.worktree_root = Some(root.join("wt"));
12846        runner.state.config.graph.advise = false;
12847        runner.roles.implementers = vec![spec("implement")];
12848        runner.roles.judges = vec![spec("judge")];
12849        runner.prep().await.unwrap();
12850        runner
12851    }
12852
12853    #[tokio::test]
12854    #[cfg(unix)]
12855    async fn parked_implementer_persists_completed_work() {
12856        for registered in [false, true] {
12857            let tmp = tempfile::tempdir().unwrap();
12858            let mut runner = registered_runner(tmp.path()).await;
12859            if !registered {
12860                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12861            }
12862            let marker = tmp.path().join("started");
12863            let release = tmp.path().join("release");
12864            let mut implementer = spec("implement");
12865            implementer.command = vec![
12866                "sh".into(),
12867                "-c".into(),
12868                format!(
12869                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12870                    marker.display(),
12871                    release.display()
12872                ),
12873            ];
12874            runner.roles.implementers = vec![implementer.clone()];
12875            runner.state.config.agents = vec![implementer];
12876            let pause = runner.pause.clone();
12877            let (result, ()) = tokio::join!(runner.implement(), async {
12878                tokio::time::timeout(Duration::from_secs(10), async {
12879                    while !marker.exists() {
12880                        tokio::time::sleep(Duration::from_millis(10)).await;
12881                    }
12882                })
12883                .await
12884                .unwrap();
12885                pause.park();
12886                std::fs::write(&release, "go").unwrap();
12887            });
12888            result.unwrap();
12889            let candidate = &runner.state.candidates[0];
12890            assert!(candidate.commits > 0, "{candidate:?}");
12891            assert!(candidate.summary.contains("saved-result"));
12892            assert_eq!(candidate.files, 1);
12893            let before = candidate.commits;
12894            runner.implement().await.unwrap();
12895            assert_eq!(runner.state.candidates[0].commits, before);
12896        }
12897    }
12898
12899    #[tokio::test]
12900    #[cfg(unix)]
12901    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12902        let tmp = tempfile::tempdir().unwrap();
12903        let mut runner = registered_runner(tmp.path()).await;
12904        let winner = runner.state.candidates[0].clone();
12905        let cache = tmp.path().join("cache");
12906        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12907        runner.state.config.graph.timeout_resource_wait = 0;
12908        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12909        let crate::cache::AcquireOutcome::Acquired(held) =
12910            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12911        else {
12912            panic!("cache must be free")
12913        };
12914        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12915            from_run: "earlier".into(),
12916            generation: 1,
12917            mode: crate::recovery::Mode::VerifyRepair,
12918            reasons: Vec::new(),
12919            reviewed_head: None,
12920            rounds: Vec::new(),
12921            open_findings: Vec::new(),
12922            evidence: None,
12923            criteria_inherited: true,
12924            repair: crate::recovery::Repair::NotStarted,
12925        });
12926        runner.recovery_repair(&winner).await.unwrap();
12927        assert!(matches!(
12928            runner.state.recovery.as_ref().unwrap().repair,
12929            crate::recovery::Repair::WaitingForRestore { .. }
12930        ));
12931        drop(held);
12932        runner.state.resource_block = None;
12933        runner.state.config.graph.timeout_resource_wait = 1;
12934        runner.recovery_repair(&winner).await.unwrap();
12935        assert!(matches!(
12936            runner.state.recovery.as_ref().unwrap().repair,
12937            crate::recovery::Repair::Done { .. }
12938        ));
12939        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12940    }
12941
12942    #[tokio::test]
12943    #[cfg(unix)]
12944    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12945        use std::os::fd::AsRawFd as _;
12946        for graft in [false, true] {
12947            let tmp = tempfile::tempdir().unwrap();
12948            let mut runner = registered_runner(tmp.path()).await;
12949            runner.state.config.graph.timeout_resource_wait = 1;
12950            let winner = runner.state.candidates[0].clone();
12951            let marker = tmp.path().join("fixer-started");
12952            let mut fixer = spec("fixer");
12953            fixer.command = vec![
12954                "sh".into(),
12955                "-c".into(),
12956                format!("echo started >> '{}'; echo done", marker.display()),
12957            ];
12958            runner.roles.fixer = Some(vec![fixer]);
12959            if graft {
12960                let mut loser = winner.clone();
12961                loser.label = 'B';
12962                loser.index = 1;
12963                runner.state.candidates.push(loser);
12964                runner.state.config.graph.graft_runner_up = true;
12965                runner.state.tally = Some(Tally {
12966                    first_choice: BTreeMap::new(),
12967                    borda: BTreeMap::new(),
12968                    winner: winner.label,
12969                    rankings: 1,
12970                    unanimous_initial: true,
12971                    deliberated: false,
12972                    changed_votes: 0,
12973                    unanimous_final: true,
12974                    tie_break: None,
12975                    judges: 1,
12976                    present: 1,
12977                    quorum: 1,
12978                    met_quorum: true,
12979                    uncontested: None,
12980                });
12981                runner.state.judgements.push(crate::run::Judgement {
12982                    judge: 1,
12983                    seat: "judge-1".into(),
12984                    agent: "judge".into(),
12985                    ranking: vec![winner.label, 'B'],
12986                    reasons: BTreeMap::new(),
12987                    confidence: None,
12988                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12989                    order: vec![0, 1],
12990                    failed: None,
12991                    duration_ms: 0,
12992                });
12993            } else {
12994                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12995                    from_run: "earlier".into(),
12996                    generation: 1,
12997                    mode: crate::recovery::Mode::VerifyRepair,
12998                    reasons: Vec::new(),
12999                    reviewed_head: None,
13000                    rounds: Vec::new(),
13001                    open_findings: Vec::new(),
13002                    evidence: None,
13003                    criteria_inherited: true,
13004                    repair: crate::recovery::Repair::NotStarted,
13005                });
13006            }
13007            let reader = crate::repo_lock::SeatLease::for_checkout(
13008                &winner.worktree,
13009                false,
13010                crate::config::test_coordination().as_ref(),
13011            )
13012            .await
13013            .unwrap();
13014            let pause = runner.pause.clone();
13015            let (result, ()) = tokio::join!(
13016                async {
13017                    if graft {
13018                        runner.graft().await
13019                    } else {
13020                        runner.recovery_repair(&winner).await
13021                    }
13022                },
13023                async {
13024                    tokio::time::sleep(Duration::from_millis(100)).await;
13025                    pause.park();
13026                }
13027            );
13028            result.unwrap();
13029            assert!(!marker.exists(), "a blocked seat must not spawn");
13030            // Exercise the persisted state, as a binary replacement/resume does.
13031            runner.state = RunState::load(&runner.state.id).unwrap();
13032            if graft {
13033                assert_eq!(
13034                    runner.state.graft.as_ref().unwrap().outcome,
13035                    GraftOutcome::WaitingForRestore
13036                );
13037            } else {
13038                assert!(matches!(
13039                    runner.state.recovery.as_ref().unwrap().repair,
13040                    crate::recovery::Repair::WaitingForRestore { .. }
13041                ));
13042            }
13043            drop(reader);
13044            runner.pause = Pause::new();
13045            runner.state.resource_block = None;
13046            // Deferred restoration must still respect an external lock owner.
13047            let external =
13048                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13049            assert_eq!(
13050                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13051                0
13052            );
13053            let leftover = winner.worktree.join("leftover");
13054            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13055            let result = if graft {
13056                runner.graft().await
13057            } else {
13058                runner.recovery_repair(&winner).await
13059            };
13060            assert!(
13061                result
13062                    .unwrap_err()
13063                    .is::<crate::repo_lock::ResourceBlocked>()
13064            );
13065            assert!(leftover.exists());
13066            assert!(!marker.exists());
13067            drop(external);
13068            runner.state.resource_block = None;
13069            let result = if graft {
13070                runner.graft().await
13071            } else {
13072                runner.recovery_repair(&winner).await
13073            };
13074            result.unwrap();
13075            assert!(!leftover.exists(), "restore must precede retry");
13076            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13077            if graft {
13078                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13079            } else {
13080                assert!(matches!(
13081                    runner.state.recovery.as_ref().unwrap().repair,
13082                    crate::recovery::Repair::Done { .. }
13083                ));
13084            }
13085        }
13086    }
13087
13088    #[tokio::test]
13089    #[cfg(unix)]
13090    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13091        use std::os::fd::AsRawFd as _;
13092        let tmp = tempfile::tempdir().unwrap();
13093        let mut runner = registered_runner(tmp.path()).await;
13094        runner.state.config.graph.timeout_resource_wait = 1;
13095        let winner = runner.state.candidates[0].clone();
13096        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13097        runner.state.tally = Some(Tally {
13098            first_choice: BTreeMap::new(),
13099            borda: BTreeMap::new(),
13100            winner: winner.label,
13101            rankings: 0,
13102            unanimous_initial: false,
13103            deliberated: false,
13104            changed_votes: 0,
13105            unanimous_final: false,
13106            tie_break: None,
13107            judges: 0,
13108            present: 0,
13109            quorum: 0,
13110            met_quorum: true,
13111            uncontested: Some("fixture".into()),
13112        });
13113        runner.state.graft = Some(GraftRecord {
13114            pre: pre.clone(),
13115            post: None,
13116            lines: None,
13117            limit: None,
13118            agent: None,
13119            outcome: GraftOutcome::Running,
13120            requested: Vec::new(),
13121            adoption: None,
13122            adoption_error: None,
13123        });
13124        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13125        git::commit_all(&winner.worktree, "unfinished graft")
13126            .await
13127            .unwrap();
13128        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13129        let leftover = winner.worktree.join("leftover");
13130        std::fs::write(&leftover, "untrusted edits").unwrap();
13131        let path = runner.state.repo.join(".git/updater.lock");
13132        let external = std::fs::File::create(&path).unwrap();
13133        assert_eq!(
13134            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13135            0
13136        );
13137        assert!(
13138            runner
13139                .graft()
13140                .await
13141                .unwrap_err()
13142                .is::<crate::repo_lock::ResourceBlocked>()
13143        );
13144        assert!(matches!(
13145            runner.state.graft.as_ref().unwrap().outcome,
13146            GraftOutcome::Running
13147        ));
13148        assert_eq!(
13149            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13150            unfinished
13151        );
13152        assert!(leftover.exists());
13153        drop(external);
13154        std::fs::remove_file(&path).unwrap();
13155        std::fs::create_dir(&path).unwrap();
13156        assert!(runner.graft().await.is_err());
13157        assert!(matches!(
13158            runner.state.graft.as_ref().unwrap().outcome,
13159            GraftOutcome::Running
13160        ));
13161        assert_eq!(
13162            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13163            unfinished
13164        );
13165        assert!(leftover.exists());
13166        std::fs::remove_dir(&path).unwrap();
13167        runner.graft().await.unwrap();
13168        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13169        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13170        assert!(!leftover.exists());
13171    }
13172
13173    #[tokio::test]
13174    #[cfg(unix)]
13175    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13176        use std::os::fd::AsRawFd as _;
13177        let tmp = tempfile::tempdir().unwrap();
13178        let mut runner = registered_runner(tmp.path()).await;
13179        runner.state.prep_complete = Some(false);
13180        runner.state.config.graph.timeout_resource_wait = 1;
13181        runner.state.save().unwrap();
13182        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13183        assert_eq!(
13184            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13185            0
13186        );
13187        runner.execute().await.unwrap();
13188        let disk = RunState::load(&runner.state.id).unwrap();
13189        assert!(disk.driver_exited);
13190        assert_eq!(disk.driver_pid, None);
13191        assert_eq!(runner.state.status, RunStatus::Blocked);
13192        assert_eq!(
13193            runner.state.resource_block.as_ref().unwrap().kind,
13194            "repository coordination"
13195        );
13196        assert!(runner.state.prep_failure.is_none());
13197        assert!(runner.state.seats.is_empty());
13198        runner.pause.park();
13199        // Include headroom for Git subprocesses under parallel test load;
13200        // the repository contention deadline itself remains one second.
13201        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13202            .await
13203            .unwrap()
13204            .unwrap();
13205        assert!(runner.state.parked);
13206        assert!(runner.state.prep_failure.is_none());
13207        let disk = RunState::load(&runner.state.id).unwrap();
13208        assert!(disk.driver_exited);
13209        assert_eq!(disk.driver_pid, None);
13210    }
13211
13212    #[tokio::test]
13213    #[cfg(unix)]
13214    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13215        use std::os::fd::AsRawFd as _;
13216        let tmp = tempfile::tempdir().unwrap();
13217        let mut runner = registered_runner(tmp.path()).await;
13218        runner.state.prep_complete = Some(false);
13219        runner.state.config.graph.timeout_resource_wait = 30;
13220        runner.state.save().unwrap();
13221        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13222        assert_eq!(
13223            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13224            0
13225        );
13226        assert!(
13227            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13228                .await
13229                .is_err()
13230        );
13231        let disk = RunState::load(&runner.state.id).unwrap();
13232        assert!(disk.driver_exited);
13233        assert_eq!(disk.driver_pid, None);
13234        assert!(disk.status.resumable());
13235        assert!(runner.state.driver_exited);
13236        assert_eq!(runner.state.driver_pid, None);
13237        runner.state.save().unwrap();
13238        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13239    }
13240
13241    #[tokio::test]
13242    #[cfg(unix)]
13243    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13244        let tmp = tempfile::tempdir().unwrap();
13245        let mut runner = registered_runner(tmp.path()).await;
13246        let winner = runner.state.candidates[0].clone();
13247        let cache = tmp.path().join("cache");
13248        let started = tmp.path().join("pre-gate-started");
13249        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13250        runner.state.config.verify.gate =
13251            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13252        let home = crate::run::home();
13253        let owner =
13254            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13255        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13256            crate::cache::AcquireOutcome::Acquired(g) => g,
13257            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13258        };
13259        let mut gate = Box::pin(runner.run_gate(&winner));
13260        assert!(
13261            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13262                .await
13263                .is_err()
13264        );
13265        assert!(!started.exists(), "pre-gate must wait for the cache too");
13266        // Simulate the wave that already owns the cache entering invoke.
13267        // It must obtain the editor reservation and finish without waiting
13268        // for the gate that is queued behind its cache.
13269        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13270        let editor = crate::repo_lock::SeatLease::checkout_with(
13271            &winner.worktree,
13272            true,
13273            crate::config::test_coordination().as_ref(),
13274            &wait,
13275        )
13276        .await
13277        .unwrap();
13278        drop(editor);
13279        guard.release();
13280        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13281            .await
13282            .unwrap()
13283            .unwrap();
13284        drop(gate);
13285        assert!(outcomes.iter().all(CommandOutcome::ok));
13286        assert!(started.exists());
13287    }
13288
13289    #[tokio::test]
13290    #[cfg(unix)]
13291    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13292        use std::os::fd::AsRawFd as _;
13293
13294        ask_test_home();
13295        let tmp = tempfile::tempdir().unwrap();
13296        let repo = tmp.path().join("repo");
13297        std::fs::create_dir_all(&repo).unwrap();
13298        init_repo(&repo);
13299        let common = repo.join(".git");
13300        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13301        let mut runner = runner_at(RunStatus::Prep);
13302        runner.state.repo = repo.clone();
13303        runner.state.config.coordination = crate::config::test_coordination();
13304        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13305        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13306        runner.state.config.graph.advise = false;
13307        runner.roles.implementers = vec![spec("implement")];
13308        runner.roles.judges = vec![spec("judge")];
13309        runner.prep().await.unwrap();
13310        let winner = runner.state.candidates[0].clone();
13311        // Emulate edits already left by a completed seat. The fixture CLI
13312        // only reports completion; it performs no repository operations.
13313        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13314        let completed = tmp.path().join("seat-completed");
13315        runner.roles.implementers[0].command = vec![
13316            "sh".into(),
13317            "-c".into(),
13318            format!("touch '{}'; echo completed", completed.display()),
13319        ];
13320        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13321        assert_eq!(
13322            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13323            0
13324        );
13325        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13326        let mut implement = Box::pin(runner.implement());
13327        tokio::select! {
13328            result = &mut implement => panic!("rescue must wait: {result:?}"),
13329            result = tokio::time::timeout(Duration::from_secs(10), async {
13330                while !completed.exists() {
13331                    tokio::time::sleep(Duration::from_millis(20)).await;
13332                }
13333            }) => result.unwrap(),
13334        }
13335        assert!(
13336            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13337                .await
13338                .is_err()
13339        );
13340        assert_eq!(
13341            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13342            before
13343        );
13344        assert_eq!(
13345            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13346            0
13347        );
13348        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13349            .await
13350            .unwrap()
13351            .unwrap();
13352        drop(implement);
13353        assert_ne!(
13354            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13355            before
13356        );
13357        assert!(git::is_clean(&winner.worktree).await.unwrap());
13358
13359        let started = tmp.path().join("pre-gate-started");
13360        let proceed = tmp.path().join("pre-gate-proceed");
13361        let checked = tmp.path().join("gate-ran");
13362        runner.state.config.verify.pre_gate = vec![format!(
13363            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13364            started.display(),
13365            proceed.display()
13366        )];
13367        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13368        assert_eq!(
13369            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13370            0
13371        );
13372        let mut gate = Box::pin(runner.run_gate(&winner));
13373        assert!(
13374            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13375                .await
13376                .is_err()
13377        );
13378        assert!(!started.exists());
13379        assert!(!checked.exists());
13380        assert!(git::is_clean(&winner.worktree).await.unwrap());
13381        assert_eq!(
13382            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13383            0
13384        );
13385        tokio::select! {
13386            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13387            result = tokio::time::timeout(Duration::from_secs(10), async {
13388                while !started.exists() {
13389                    tokio::time::sleep(Duration::from_millis(20)).await;
13390                }
13391            }) => result.unwrap(),
13392        }
13393        assert_eq!(
13394            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13395            -1,
13396            "mechanical edits must hold the real lock"
13397        );
13398        std::fs::write(proceed, "ready").unwrap();
13399        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13400            .await
13401            .unwrap()
13402            .unwrap();
13403        drop(gate);
13404        assert!(outcomes.iter().all(CommandOutcome::ok));
13405        assert!(checked.exists());
13406        assert!(runner.state.pre_gate_commit.is_some());
13407        assert!(git::is_clean(&winner.worktree).await.unwrap());
13408        assert_eq!(
13409            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13410            0,
13411            "gate completion must release coordination"
13412        );
13413        drop(outsider);
13414        std::fs::remove_file(common.join("updater.lock")).unwrap();
13415        std::fs::create_dir(common.join("updater.lock")).unwrap();
13416        std::fs::remove_file(&started).unwrap();
13417        std::fs::remove_file(&checked).unwrap();
13418        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13419        let error = runner.run_gate(&winner).await.unwrap_err();
13420        assert!(
13421            error
13422                .to_string()
13423                .contains("acquire mandatory repository lock")
13424        );
13425        assert!(
13426            !started.exists(),
13427            "failed acquisition must prevent pre-gate"
13428        );
13429        assert!(!checked.exists(), "failed acquisition must prevent gate");
13430        assert!(git::is_clean(&winner.worktree).await.unwrap());
13431        assert_eq!(
13432            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13433            head
13434        );
13435    }
13436
13437    #[tokio::test]
13438    #[cfg(unix)]
13439    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13440        use std::os::fd::AsRawFd as _;
13441        use std::time::Duration;
13442
13443        ask_test_home();
13444        let tmp = tempfile::tempdir().unwrap();
13445        let repo = tmp.path().join("repo");
13446        std::fs::create_dir_all(&repo).unwrap();
13447        init_repo(&repo);
13448        let common = repo.join(".git");
13449        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13450        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13451        assert_eq!(
13452            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13453            0
13454        );
13455        let mut runner = runner_at(RunStatus::Prep);
13456        runner.state.repo = repo.clone();
13457        runner.state.config.coordination = crate::config::test_coordination();
13458        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13459        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13460        runner.state.config.graph.advise = true;
13461        runner.state.config.graph.advisors = 1;
13462        runner.roles.implementers = vec![spec("implement")];
13463        runner.roles.judges = vec![spec("judge")];
13464        let root = runner.state.worktree_root();
13465        let started = tmp.path().join("setup-started");
13466        let proceed = tmp.path().join("setup-proceed");
13467        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13468            run: Some(format!(
13469                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13470                started.display(),
13471                proceed.display()
13472            )),
13473            ..Default::default()
13474        }];
13475        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13476        let config = git::git(&repo, &["config", "--local", "--list"])
13477            .await
13478            .unwrap();
13479        let mut prep = Box::pin(runner.prep());
13480        assert!(
13481            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13482                .await
13483                .is_err()
13484        );
13485        assert!(
13486            !root.exists(),
13487            "contention must prevent even checkout creation"
13488        );
13489        assert!(!started.exists(), "contention must prevent setup");
13490        assert_eq!(
13491            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13492            branches
13493        );
13494        assert_eq!(
13495            git::git(&repo, &["config", "--local", "--list"])
13496                .await
13497                .unwrap(),
13498            config
13499        );
13500        drop(lock);
13501        tokio::select! {
13502            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13503            result = tokio::time::timeout(Duration::from_secs(10), async {
13504                while !started.exists() {
13505                    tokio::time::sleep(Duration::from_millis(20)).await;
13506                }
13507            }) => result.unwrap(),
13508        }
13509        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13510        assert_eq!(
13511            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13512            -1
13513        );
13514        assert!(
13515            tokio::time::timeout(
13516                Duration::from_millis(100),
13517                crate::repo_lock::SeatLease::for_checkout(
13518                    &repo,
13519                    true,
13520                    crate::config::test_coordination().as_ref()
13521                )
13522            )
13523            .await
13524            .is_err(),
13525            "preparation must also exclude local editors"
13526        );
13527        std::fs::write(proceed, "ready").unwrap();
13528        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13529            .await
13530            .unwrap()
13531            .unwrap();
13532        drop(prep);
13533        assert_eq!(runner.state.prep_complete, Some(true));
13534        assert!(root.join("judge-1").exists());
13535        assert!(root.join("advisor-1").exists());
13536        assert_eq!(
13537            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13538            0
13539        );
13540    }
13541
13542    #[tokio::test]
13543    #[cfg(unix)]
13544    async fn preparation_lock_failure_prevents_repository_changes() {
13545        ask_test_home();
13546        let tmp = tempfile::tempdir().unwrap();
13547        let repo = tmp.path().join("repo");
13548        std::fs::create_dir_all(&repo).unwrap();
13549        init_repo(&repo);
13550        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13551        // A directory at the required lock path makes opening the real lock
13552        // fail, rather than merely exercising an invalid registration.
13553        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13554        let mut runner = runner_at(RunStatus::Prep);
13555        runner.state.repo = repo.clone();
13556        runner.state.config.coordination = crate::config::test_coordination();
13557        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13558        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13559        runner.roles.implementers = vec![spec("implement")];
13560        let error = runner.prep().await.unwrap_err();
13561        assert!(
13562            error
13563                .to_string()
13564                .contains("acquire mandatory repository lock")
13565        );
13566        assert!(!runner.state.worktree_root().exists());
13567        assert!(runner.state.candidates.is_empty());
13568        assert!(!repo.join(".git/config.worktree").exists());
13569        assert_eq!(
13570            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13571            "* main"
13572        );
13573    }
13574
13575    #[tokio::test]
13576    async fn preparation_records_creation_intent_before_git_can_fail() {
13577        ask_test_home();
13578        let tmp = tempfile::tempdir().unwrap();
13579        let repo = tmp.path().join("repo");
13580        std::fs::create_dir_all(&repo).unwrap();
13581        init_repo(&repo);
13582        let mut runner = runner_at(RunStatus::Prep);
13583        runner.state.repo = repo.clone();
13584        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13585        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13586        runner.state.config.blind.commit_msg_hook = false;
13587        runner.roles.implementers = vec![spec("one")];
13588        let label = blind::assign_labels(1, runner.state.seed)[0];
13589        let branch = runner.state.branch_for(label);
13590        // A conflicting branch makes Git refuse creation before a candidate
13591        // can be registered. The durable intent must still identify it.
13592        git::git(&repo, &["branch", &branch]).await.unwrap();
13593        assert!(runner.prep().await.is_err());
13594        let saved = RunState::load(&runner.state.id).unwrap();
13595        assert!(saved.candidates.is_empty());
13596        let event = saved
13597            .events
13598            .iter()
13599            .find(|e| e.node == "prep-intent")
13600            .unwrap();
13601        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13602        assert_eq!(intent["branch"], branch);
13603        assert_eq!(intent["start"], runner.state.base_commit);
13604        assert_eq!(
13605            intent["worktree"],
13606            runner
13607                .state
13608                .worktree_root()
13609                .join(format!("cand-{label}"))
13610                .to_string_lossy()
13611                .as_ref()
13612        );
13613        assert_eq!(intent["picks"], serde_json::json!([]));
13614    }
13615
13616    #[tokio::test]
13617    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13618        ask_test_home();
13619        let tmp = tempfile::tempdir().unwrap();
13620        let repo = tmp.path().join("repo");
13621        std::fs::create_dir_all(&repo).unwrap();
13622        init_repo(&repo);
13623        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13624        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13625        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13626        git::git(&repo, &["add", "."]).await.unwrap();
13627        git::git(&repo, &["commit", "-m", "required seed"])
13628            .await
13629            .unwrap();
13630        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13631        git::git(&repo, &["checkout", "main"]).await.unwrap();
13632        let mut runner = runner_at(RunStatus::Prep);
13633        runner.state.repo = repo.clone();
13634        runner.state.base_commit = base.clone();
13635        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13636        runner.state.config.graph.advise = false;
13637        runner.state.config.blind.commit_msg_hook = false;
13638        runner.state.seeds = vec![refs::Seed {
13639            token: seed.clone(),
13640            sha: seed.clone(),
13641            kind: refs::SeedKind::Unmerged,
13642            branch: false,
13643            detail: String::new(),
13644        }];
13645        runner.roles.implementers = vec![spec("one"), spec("two")];
13646        runner.roles.judges = vec![spec("judge")];
13647        let label = blind::assign_labels(2, runner.state.seed)[0];
13648        let branch = runner.state.branch_for(label);
13649        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13650        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13651            .await
13652            .unwrap();
13653        runner.state.candidates.push(Candidate {
13654            index: 0,
13655            label,
13656            agent: "one".into(),
13657            branch,
13658            worktree,
13659            summary: String::new(),
13660            stat: String::new(),
13661            files: 0,
13662            commits: 0,
13663            empty: false,
13664            failed: None,
13665            verified_noop: None,
13666            duration_ms: 0,
13667            folded: false,
13668        });
13669        runner.state.save().unwrap();
13670        // Reload the checkpoint immediately before its first cherry-pick.
13671        runner.state = RunState::load(&runner.state.id).unwrap();
13672        assert_eq!(runner.state.prep_complete, Some(false));
13673        runner.prep().await.unwrap();
13674        assert_eq!(runner.state.prep_complete, Some(true));
13675        assert_eq!(runner.state.candidates.len(), 2);
13676        for candidate in &runner.state.candidates {
13677            assert_eq!(
13678                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13679                git::tree_of(&repo, &seed).await.unwrap()
13680            );
13681            assert_eq!(
13682                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13683                "required commit"
13684            );
13685        }
13686        assert!(runner.state.worktree_root().join("judge-1").exists());
13687        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13688        runner.prep().await.unwrap();
13689        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13690
13691        runner.state.prep_complete = Some(false);
13692        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13693            run: Some(format!(
13694                "test -f '{}'",
13695                tmp.path().join("dependency-ready").display()
13696            )),
13697            ..Default::default()
13698        }];
13699        assert!(runner.prep().await.is_err());
13700        assert!(runner.state.prep_failure.is_some());
13701        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13702        runner.state = RunState::load(&runner.state.id).unwrap();
13703        assert!(
13704            runner.prep().await.is_err(),
13705            "correcting the cause alone must not auto-retry"
13706        );
13707        let failed_at = runner.state.events.last().unwrap().at;
13708        assert!(
13709            runner.retry_preparation(failed_at).is_err(),
13710            "old approvals cannot unlock the hold"
13711        );
13712        let mut task = crate::queue::Task::new(
13713            "retry".into(),
13714            "retry setup".into(),
13715            repo.clone(),
13716            crate::queue::Source::Human,
13717        );
13718        task.start(runner.state.id.clone());
13719        task.hold_machine(Some("setup failed".into()));
13720        task.release_operator("cli", None);
13721        runner
13722            .retry_preparation(task.resumption.as_ref().unwrap().at)
13723            .unwrap();
13724        runner.prep().await.unwrap();
13725        assert!(runner.state.prep_failure.is_none());
13726        assert_eq!(runner.state.prep_complete, Some(true));
13727        assert_eq!(runner.state.candidates.len(), 2);
13728        assert!(runner.state.worktree_root().join("judge-1").exists());
13729        for candidate in &runner.state.candidates {
13730            assert_eq!(
13731                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13732                git::tree_of(&repo, &seed).await.unwrap()
13733            );
13734        }
13735    }
13736
13737    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13738        let mut heads = Vec::new();
13739        for candidate in candidates {
13740            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13741        }
13742        heads
13743    }
13744
13745    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13746        ask_test_home();
13747        let tmp = tempfile::tempdir().unwrap();
13748        let repo = tmp.path().join("repo");
13749        std::fs::create_dir(&repo).unwrap();
13750        init_repo(&repo);
13751        let mut runner = runner_at(RunStatus::Reviewing);
13752        runner.state.repo = repo.clone();
13753        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13754        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13755        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13756            let branch = format!("candidate-{label}");
13757            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13758            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13759                .await
13760                .unwrap();
13761            if label == 'B' {
13762                std::fs::write(worktree.join("change"), "winner").unwrap();
13763                git::rescue_commit(&worktree, "candidate change")
13764                    .await
13765                    .unwrap();
13766            }
13767            runner.state.candidates.push(Candidate {
13768                index,
13769                label,
13770                agent: "alpha".into(),
13771                branch,
13772                worktree,
13773                summary: "completed reply".into(),
13774                stat: String::new(),
13775                files: usize::from(label == 'B'),
13776                commits: usize::from(label == 'B'),
13777                empty: label == 'A',
13778                failed: None,
13779                verified_noop: None,
13780                duration_ms: 1,
13781                folded: false,
13782            });
13783        }
13784        runner.state.judge_skipped = true;
13785        runner.state.tally = Some(Tally {
13786            first_choice: BTreeMap::new(),
13787            borda: BTreeMap::new(),
13788            winner: 'B',
13789            rankings: 0,
13790            unanimous_initial: false,
13791            deliberated: false,
13792            changed_votes: 0,
13793            unanimous_final: false,
13794            tie_break: None,
13795            judges: 0,
13796            present: 0,
13797            quorum: 0,
13798            met_quorum: true,
13799            uncontested: Some("only B produced a change".into()),
13800        });
13801        runner.state.save().unwrap();
13802        (tmp, runner)
13803    }
13804
13805    #[tokio::test]
13806    #[cfg(unix)]
13807    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13808        for (solo, failed_checkpoint) in
13809            [(true, false), (false, false), (true, true), (false, true)]
13810        {
13811            let (_tmp, mut runner) = preservation_resume_fixture().await;
13812            if solo {
13813                runner.state.candidates.retain(|c| c.label == 'A');
13814            }
13815            runner.state.candidates[0].empty = false;
13816            runner.state.candidates[0].summary.clear();
13817            runner.state.tally = None;
13818            runner.state.judge_skipped = false;
13819            runner.state.status = RunStatus::Implementing;
13820            runner.state.config.disk.min_free_bytes = u64::MAX;
13821            let mut implementer = spec("alpha");
13822            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13823            runner.roles.implementers = vec![implementer];
13824            let patch = runner.state.dir().join("artifacts/cand-A.patch");
13825            if failed_checkpoint {
13826                std::fs::create_dir_all(&patch).unwrap();
13827            }
13828            runner.state.save().unwrap();
13829            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13830            runner.state = RunState::load(&runner.state.id).unwrap();
13831            assert_eq!(runner.state.status, RunStatus::Blocked);
13832            assert!(runner.state.interrupted_implementations.contains(&'A'));
13833            assert_eq!(runner.state.candidates[0].commits, 0);
13834            assert!(!runner.state.candidates[0].empty);
13835            assert!(runner.state.candidates[0].failed.is_none());
13836            if failed_checkpoint {
13837                assert!(runner.state.candidate_preservation.contains_key(&'A'));
13838                std::fs::remove_dir(patch).unwrap();
13839            }
13840            runner.state.resource_block = None;
13841            runner.state.config.disk.min_free_bytes = 0;
13842            runner.roles.implementers[0].command = vec![
13843                "sh".into(),
13844                "-c".into(),
13845                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13846            ];
13847            runner.implement().await.unwrap();
13848            assert!(runner.state.interrupted_implementations.is_empty());
13849            assert!(runner.state.candidate_preservation.is_empty());
13850            assert_eq!(runner.state.candidates[0].commits, 1);
13851            assert!(!runner.state.candidates[0].empty);
13852            assert!(
13853                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13854                    .unwrap()
13855                    .contains("retried empty interrupted seat")
13856            );
13857            if !solo {
13858                assert_eq!(runner.state.candidates[1].commits, 1);
13859            }
13860        }
13861    }
13862
13863    #[tokio::test]
13864    #[cfg(unix)]
13865    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13866        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13867            let (_tmp, mut runner) = preservation_resume_fixture().await;
13868            runner.state.candidates.retain(|c| c.label == 'A');
13869            runner.state.candidates[0].empty = false;
13870            runner.state.candidates[0].summary.clear();
13871            runner.state.tally = None;
13872            runner.state.judge_skipped = false;
13873            runner.state.status = RunStatus::Implementing;
13874            // Force the real post-spawn watchdog, independently of available
13875            // disk space. Dirty/committed work predates its first measurement.
13876            runner.state.config.disk.min_free_bytes = u64::MAX;
13877            let wt = runner.state.candidates[0].worktree.clone();
13878            std::fs::write(
13879                wt.join("interrupted-source"),
13880                "retain implementation work\n",
13881            )
13882            .unwrap();
13883            if committed {
13884                git::rescue_commit(&wt, "implementation checkpoint")
13885                    .await
13886                    .unwrap();
13887            }
13888            let lock = git::git(
13889                &wt,
13890                &[
13891                    "rev-parse",
13892                    "--path-format=absolute",
13893                    "--git-path",
13894                    "index.lock",
13895                ],
13896            )
13897            .await
13898            .unwrap();
13899            if block_rescue {
13900                std::fs::write(&lock, "fixture-owned lock").unwrap();
13901            }
13902            let mut implementer = spec("alpha");
13903            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13904            runner.roles.implementers = vec![implementer];
13905            runner.state.save().unwrap();
13906            let result = runner.implement().await;
13907            assert_eq!(result.is_err(), block_rescue);
13908            assert_eq!(
13909                runner.state.resource_block.as_ref().unwrap().kind,
13910                "disk pressure"
13911            );
13912            assert!(
13913                runner
13914                    .state
13915                    .events
13916                    .iter()
13917                    .any(|e| e.message.contains("agent execution interrupted"))
13918            );
13919            assert!(
13920                runner
13921                    .state
13922                    .dir()
13923                    .join("artifacts/candidate-A-recovery/staged.patch")
13924                    .exists()
13925            );
13926            if block_rescue {
13927                let saved = RunState::load(&runner.state.id).unwrap();
13928                assert!(saved.candidate_preservation.contains_key(&'A'));
13929                assert!(!saved.candidates[0].empty);
13930                assert_eq!(
13931                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
13932                    "retain implementation work\n"
13933                );
13934                std::fs::remove_file(lock).unwrap();
13935                runner.state = saved;
13936            } else {
13937                assert_eq!(runner.state.status, RunStatus::Blocked);
13938                assert_eq!(runner.state.candidates[0].commits, 1);
13939            }
13940            // Recovery must save the candidate before deciding whether another
13941            // implementation is needed. No implementer is available on retry.
13942            runner.roles.implementers.clear();
13943            runner.state.config.disk.min_free_bytes = 0;
13944            runner.state.resource_block = None;
13945            runner.implement().await.unwrap();
13946            assert!(runner.state.candidate_preservation.is_empty());
13947            assert_eq!(runner.state.candidates[0].commits, 1);
13948            assert!(!runner.state.candidates[0].empty);
13949            assert!(git::is_clean(&wt).await.unwrap());
13950            assert!(
13951                crate::run::read_artifact(&runner.state, "cand-A.patch")
13952                    .unwrap()
13953                    .contains("retain implementation work")
13954            );
13955        }
13956    }
13957
13958    #[tokio::test]
13959    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13960        let (_tmp, mut runner) = preservation_resume_fixture().await;
13961        // Schema 15 already confirmed the unfolded no-change result.
13962        runner.implement().await.unwrap();
13963        assert!(runner.state.candidate_preservation.is_empty());
13964        assert!(
13965            !runner
13966                .state
13967                .dir()
13968                .join("artifacts/candidate-A-recovery")
13969                .exists()
13970        );
13971        runner.fold_losers().await.unwrap();
13972        assert!(!runner.state.candidates[0].worktree.exists());
13973        for schema in [14, crate::run::SCHEMA] {
13974            runner.state.schema = schema;
13975            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13976            runner.state.save().unwrap();
13977            runner.state = RunState::load(&runner.state.id).unwrap();
13978            runner.implement().await.unwrap();
13979            assert!(runner.state.candidate_preservation.is_empty());
13980            assert!(runner.state.candidates[0].folded);
13981            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13982        }
13983    }
13984
13985    #[tokio::test]
13986    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13987        let (_tmp, mut runner) = preservation_resume_fixture().await;
13988        let wt = runner.state.candidates[0].worktree.clone();
13989        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13990        runner.state.schema = 14;
13991        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13992        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13993        let tally = serde_json::to_value(&runner.state.tally).unwrap();
13994        for _ in 0..2 {
13995            assert!(runner.implement().await.is_err());
13996            assert_eq!(runner.state.status, RunStatus::Blocked);
13997            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13998            assert!(
13999                runner.state.candidates[0].empty,
14000                "old verdict must remain unusable"
14001            );
14002            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14003            assert!(runner.state.judge_skipped);
14004            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14005            assert!(
14006                crate::run::read_artifact(&runner.state, "cand-A.patch")
14007                    .unwrap()
14008                    .contains("owned edits")
14009            );
14010            assert!(wt.exists());
14011            assert_eq!(
14012                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14013                    .await
14014                    .unwrap(),
14015                1
14016            );
14017            runner.state = RunState::load(&runner.state.id).unwrap();
14018        }
14019    }
14020
14021    #[tokio::test]
14022    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14023        let (_tmp, mut runner) = preservation_resume_fixture().await;
14024        let wt = runner.state.candidates[0].worktree.clone();
14025        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14026        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14027        let index_lock = PathBuf::from(
14028            git::git(
14029                &wt,
14030                &[
14031                    "rev-parse",
14032                    "--path-format=absolute",
14033                    "--git-path",
14034                    "index.lock",
14035                ],
14036            )
14037            .await
14038            .unwrap(),
14039        );
14040        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14041        runner.state.schema = 14;
14042        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14043        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14044        assert!(runner.implement().await.is_err());
14045        assert!(runner.after_implement().is_err());
14046        assert!(runner.fold_losers().await.is_err());
14047        assert_eq!(
14048            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14049            "failed seat edits"
14050        );
14051        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14052        runner.state = RunState::load(&runner.state.id).unwrap();
14053        std::fs::remove_file(index_lock).unwrap();
14054        // Recovery finds changes that the old uncontested result never judged.
14055        assert!(runner.implement().await.is_err());
14056        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14057        assert!(runner.state.candidates[0].empty);
14058        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14059        assert!(
14060            crate::run::read_artifact(&runner.state, "cand-A.patch")
14061                .unwrap()
14062                .contains("failed seat edits")
14063        );
14064        assert!(wt.exists());
14065    }
14066
14067    #[tokio::test]
14068    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14069        let (_tmp, mut runner) = preservation_resume_fixture().await;
14070        let repo = runner.state.repo.clone();
14071        let common = git::git(
14072            &repo,
14073            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14074        )
14075        .await
14076        .unwrap();
14077        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14078        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14079        assert!(runner.fold_losers().await.is_err());
14080        assert!(!runner.state.candidates[0].worktree.exists());
14081        assert!(!runner.state.candidates[0].folded);
14082        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14083        runner.state = RunState::load(&runner.state.id).unwrap();
14084        std::fs::remove_file(branch_lock).unwrap();
14085        let judge = runner.state.worktree_root().join("judge-1");
14086        git::worktree_add_detached(&repo, &judge, "main")
14087            .await
14088            .unwrap();
14089        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14090        runner.roles.judges = vec![spec("judge")];
14091        assert!(runner.fold_losers().await.is_err());
14092        runner.state = RunState::load(&runner.state.id).unwrap();
14093        assert!(
14094            runner.state.candidates[0].folded,
14095            "candidate progress saved before judge failure"
14096        );
14097        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14098        assert!(judge.join("uncommitted").exists());
14099        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14100        runner.fold_losers().await.unwrap();
14101        assert!(!judge.exists());
14102        runner.fold_losers().await.unwrap();
14103    }
14104
14105    #[tokio::test]
14106    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14107        let (_tmp, mut runner) = preservation_resume_fixture().await;
14108        runner.state.candidates.retain(|c| c.label == 'B');
14109        runner.state.tally = None;
14110        runner.state.judge_skipped = false;
14111        runner.state.prep_complete = Some(true);
14112        runner.state.config.graph.advise = false;
14113        runner.state.config.graph.selected_implementer = Some("alpha".into());
14114        let wt = runner.state.candidates[0].worktree.clone();
14115        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14116        runner
14117            .state
14118            .candidate_preservation
14119            .insert('B', "awaiting persistence".into());
14120        let index_lock = git::git(
14121            &wt,
14122            &[
14123                "rev-parse",
14124                "--path-format=absolute",
14125                "--git-path",
14126                "index.lock",
14127            ],
14128        )
14129        .await
14130        .unwrap();
14131        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14132        runner.state.save().unwrap();
14133        assert!(runner.execute_graph().await.is_err());
14134        assert_eq!(runner.state.status, RunStatus::Blocked);
14135        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14136        std::fs::remove_file(index_lock).unwrap();
14137        // Park at the next graph boundary: this tests the real selected-seat
14138        // hold guard without dispatching reviewers or publishing anything.
14139        for _ in 0..2 {
14140            runner.state = RunState::load(&runner.state.id).unwrap();
14141            runner.pause.park();
14142            runner.execute_graph().await.unwrap();
14143            assert_eq!(runner.state.status, RunStatus::Implementing);
14144            assert!(runner.state.candidate_preservation.is_empty());
14145            assert_eq!(runner.state.candidates[0].commits, 2);
14146            assert!(runner.state.tally.is_none());
14147        }
14148        runner.implement().await.unwrap();
14149        assert_eq!(runner.state.status, RunStatus::Judging);
14150        // A genuine selected-implementer hold must still require release.
14151        runner.state.status = RunStatus::Blocked;
14152        runner.state.candidates[0].failed = Some("implementation failed".into());
14153        runner.state.save().unwrap();
14154        let error = runner.execute_graph().await.unwrap_err();
14155        assert!(
14156            error
14157                .to_string()
14158                .contains("selected implementation is held")
14159        );
14160    }
14161
14162    #[tokio::test]
14163    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14164        let (_tmp, mut runner) = preservation_resume_fixture().await;
14165        let repo = runner.state.repo.clone();
14166        let candidate = runner.state.candidates[0].clone();
14167        // Remove only the disposable fixture directory, leaving Git's linked
14168        // worktree registration and branch intact, as in a vanished checkout.
14169        let registered = git::worktree_holding(&repo, &candidate.branch)
14170            .await
14171            .unwrap()
14172            .unwrap();
14173        assert_eq!(
14174            registered.canonicalize().unwrap(),
14175            candidate.worktree.canonicalize().unwrap()
14176        );
14177        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14178        assert_eq!(
14179            git::worktree_holding(&repo, &candidate.branch)
14180                .await
14181                .unwrap(),
14182            Some(registered)
14183        );
14184        assert!(
14185            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14186            "the stale registration initially keeps the branch checked out"
14187        );
14188        runner.fold_losers().await.unwrap();
14189        assert!(runner.state.candidates[0].folded);
14190        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14191        assert_eq!(
14192            git::worktree_holding(&repo, &candidate.branch)
14193                .await
14194                .unwrap(),
14195            None
14196        );
14197        runner.state = RunState::load(&runner.state.id).unwrap();
14198        runner.fold_losers().await.unwrap();
14199    }
14200
14201    #[tokio::test]
14202    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14203        let (_tmp, mut runner) = preservation_resume_fixture().await;
14204        let repo = runner.state.repo.clone();
14205        let candidate = runner.state.candidates[0].clone();
14206        let winner = runner.state.candidates[1].clone();
14207        let registered = git::worktree_holding(&repo, &candidate.branch)
14208            .await
14209            .unwrap()
14210            .unwrap();
14211        assert_eq!(
14212            registered.canonicalize().unwrap(),
14213            candidate.worktree.canonicalize().unwrap()
14214        );
14215        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14216        assert_eq!(
14217            git::worktree_holding(&repo, &candidate.branch)
14218                .await
14219                .unwrap(),
14220            Some(registered)
14221        );
14222        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14223
14224        for _ in 0..2 {
14225            fold_run(&mut runner.state, false, &crate::run::home())
14226                .await
14227                .unwrap();
14228            runner.state = RunState::load(&runner.state.id).unwrap();
14229            assert!(runner.state.candidates[0].folded);
14230            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14231            assert_eq!(
14232                git::worktree_holding(&repo, &candidate.branch)
14233                    .await
14234                    .unwrap(),
14235                None
14236            );
14237            assert!(!runner.state.candidates[1].folded);
14238            assert!(winner.worktree.exists());
14239            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14240        }
14241    }
14242
14243    #[tokio::test]
14244    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14245        ask_test_home();
14246        let tmp = tempfile::tempdir().unwrap();
14247        let repo = tmp.path().join("repo");
14248        std::fs::create_dir(&repo).unwrap();
14249        init_repo(&repo);
14250        let mut runner = runner_at(RunStatus::Implementing);
14251        runner.state.repo = repo.clone();
14252        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14253        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14254        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14255            let branch = format!("candidate-{label}");
14256            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14257            git::worktree_add_branch(&repo, &wt, &branch, "main")
14258                .await
14259                .unwrap();
14260            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14261            git::git(&wt, &["add", "README.md"]).await.unwrap();
14262            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14263            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14264            runner.state.candidates.push(Candidate {
14265                index,
14266                label,
14267                agent: "alpha".to_owned(),
14268                branch,
14269                worktree: wt,
14270                summary: "completed implementation".to_owned(),
14271                stat: String::new(),
14272                files: 0,
14273                commits: 0,
14274                empty: false,
14275                failed: None,
14276                verified_noop: None,
14277                duration_ms: 42,
14278                folded: false,
14279            });
14280            runner
14281                .state
14282                .candidate_preservation
14283                .insert(label, "awaiting persistence".to_owned());
14284        }
14285        let wt = runner.state.candidates[0].worktree.clone();
14286        let index_lock = git::git(
14287            &wt,
14288            &[
14289                "rev-parse",
14290                "--path-format=absolute",
14291                "--git-path",
14292                "index.lock",
14293            ],
14294        )
14295        .await
14296        .unwrap();
14297        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14298        runner.state.save().unwrap();
14299        assert!(runner.persist_candidates().await.is_err());
14300        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14301        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14302        assert_eq!(
14303            runner.state.candidates[1].commits, 1,
14304            "other results are still collected"
14305        );
14306        assert!(
14307            !runner.state.candidates[0].empty,
14308            "failed rescue is not no change"
14309        );
14310        assert_eq!(
14311            runner.state.candidates[0].summary,
14312            "completed implementation"
14313        );
14314        assert!(runner.after_implement().is_err());
14315        let home = tmp.path().join("fold-home");
14316        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14317        assert!(wt.join("new-file").exists());
14318        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14319        assert!(recovery.join("staged.patch").exists());
14320        assert!(recovery.join("unstaged.patch").exists());
14321        assert_eq!(
14322            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14323            "untracked edits\n"
14324        );
14325        runner.state.begin_pass();
14326        runner.state.end_pass_parked(ParkCause::Upgrade {
14327            from: "1.0".into(),
14328            to: Some("1.1".into()),
14329        });
14330        runner.state.save().unwrap();
14331        runner.state = RunState::load(&runner.state.id).unwrap();
14332        assert!(matches!(
14333            runner.state.passes.last().unwrap().end,
14334            Some(crate::run::PassEnd::Parked {
14335                cause: ParkCause::Upgrade { .. }
14336            })
14337        ));
14338        std::fs::remove_file(index_lock).unwrap();
14339        // No implementer is configured: a persistence retry cannot dispatch a seat.
14340        runner.implement().await.unwrap();
14341        assert!(runner.state.candidate_preservation.is_empty());
14342        assert!(
14343            runner
14344                .state
14345                .candidates
14346                .iter()
14347                .all(|c| c.commits == 1 && !c.empty)
14348        );
14349        assert!(git::is_clean(&wt).await.unwrap());
14350        assert!(
14351            crate::run::read_artifact(&runner.state, "cand-A.patch")
14352                .unwrap()
14353                .contains("later tracked edits")
14354        );
14355    }
14356
14357    #[tokio::test]
14358    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14359        ask_test_home();
14360        let tmp = tempfile::tempdir().unwrap();
14361        let repo = tmp.path().join("repo");
14362        std::fs::create_dir(&repo).unwrap();
14363        init_repo(&repo);
14364        let wt = tmp.path().join("candidate");
14365        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14366            .await
14367            .unwrap();
14368        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14369        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14370        assert!(wt.join("uncommitted").exists());
14371        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14372        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14373        let mut runner = runner_at(RunStatus::Implementing);
14374        runner.state.repo = repo.clone();
14375        runner.state.base_commit = "missing-base".to_owned();
14376        runner.state.candidates.push(Candidate {
14377            index: 0,
14378            label: 'A',
14379            agent: "alpha".into(),
14380            branch: "candidate".into(),
14381            worktree: wt.clone(),
14382            summary: "completed".into(),
14383            stat: String::new(),
14384            files: 0,
14385            commits: 0,
14386            empty: false,
14387            failed: None,
14388            verified_noop: None,
14389            duration_ms: 1,
14390            folded: false,
14391        });
14392        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14393        assert!(
14394            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14395                .await
14396                .is_err()
14397        );
14398        assert!(!runner.state.candidates[0].folded);
14399        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14400        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14401        runner
14402            .state
14403            .candidate_preservation
14404            .insert('A', "pending".into());
14405        assert!(runner.persist_candidates().await.is_err());
14406        assert!(!runner.state.candidates[0].empty);
14407        assert!(!runner.state.candidates[0].folded);
14408        assert!(wt.exists());
14409        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14410        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14411        std::fs::create_dir_all(&patch_path).unwrap();
14412        assert!(
14413            runner.persist_candidates().await.is_err(),
14414            "artifact failure stays pending"
14415        );
14416        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14417        assert!(!runner.state.candidates[0].empty);
14418        std::fs::remove_dir(patch_path).unwrap();
14419        runner.persist_candidates().await.unwrap();
14420        assert!(
14421            runner.state.candidates[0].empty,
14422            "only successful inspection confirms no change"
14423        );
14424    }
14425
14426    fn init_repo(dir: &Path) {
14427        let run = |args: &[&str]| {
14428            let out = std::process::Command::new("git")
14429                .args(args)
14430                .current_dir(dir)
14431                .quiet()
14432                .output()
14433                .expect("spawn git");
14434            assert!(
14435                out.status.success(),
14436                "git {args:?} failed: {}",
14437                String::from_utf8_lossy(&out.stderr)
14438            );
14439        };
14440        run(&["init", "-b", "main"]);
14441        run(&["config", "user.name", "magi test"]);
14442        run(&["config", "user.email", "magi@example.com"]);
14443        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14444        run(&["add", "-A"]);
14445        run(&["commit", "-m", "init"]);
14446    }
14447
14448    // `settle_questions` is what closes the ghost the phone showed: a run's
14449    // seat asked something, the run then ended, and nothing was left to
14450    // abandon the question it left `open`. `HOME` is a process-wide
14451    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14452    // first time it runs in the binary — every test below still reaches the
14453    // same directory whichever call won, and each gets its own run id from
14454    // `RunState::new`, so they never collide there.
14455    fn ask_test_home() {
14456        crate::run::pin_test_home();
14457    }
14458
14459    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14460    /// reads nothing else off it.
14461    fn runner_at(status: RunStatus) -> Runner {
14462        let mut state = RunState::new(
14463            PathBuf::from("/nonexistent/repo"),
14464            "main".to_owned(),
14465            "deadbeef".to_owned(),
14466            "task".to_owned(),
14467            Config::default(),
14468        );
14469        // These fixtures share one home and save concurrently. Short clock
14470        // IDs can collide, making an ownership test read another test's run.
14471        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14472        state.id = format!(
14473            "20000101-000000-{:04x}",
14474            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14475        );
14476        state.status = status;
14477        Runner {
14478            state,
14479            roles: ResolvedRoles {
14480                implementers: Vec::new(),
14481                judges: Vec::new(),
14482                reviewers: Vec::new(),
14483                fixer: None,
14484                conductor: conductor(),
14485                implementer_roster: Vec::new(),
14486                judge_roster: Vec::new(),
14487                reviewer_roster: Vec::new(),
14488            },
14489            sem: Arc::new(Semaphore::new(1)),
14490            pause: Pause::new(),
14491            interrupt: Pause::new(),
14492        }
14493    }
14494
14495    #[test]
14496    fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14497        crate::run::pin_test_home();
14498        let mut runner = runner_at(RunStatus::Gating);
14499        runner.state.save().unwrap();
14500        // run.json can commit, but the subsequent lease metadata write fails.
14501        // Cleanup must run after releasing the short ownership lock.
14502        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14503        assert!(runner.claim_driver().is_err());
14504        assert!(runner.state.driver_exited);
14505        assert_eq!(runner.state.driver_pid, None);
14506        let disk = RunState::load(&runner.state.id).unwrap();
14507        assert!(disk.driver_exited);
14508        assert_eq!(disk.driver_pid, None);
14509        assert_eq!(disk.status, RunStatus::Gating);
14510    }
14511
14512    #[tokio::test]
14513    async fn an_early_graph_error_releases_its_driver() {
14514        crate::run::pin_test_home();
14515        let mut runner = runner_at(RunStatus::Blocked);
14516        runner.state.config.graph.selected_implementer = Some("selected".into());
14517        runner.state.save().unwrap();
14518        assert!(runner.execute().await.is_err());
14519        let disk = RunState::load(&runner.state.id).unwrap();
14520        assert!(disk.driver_exited);
14521        assert_eq!(disk.driver_pid, None);
14522        assert_eq!(disk.status, RunStatus::Blocked);
14523    }
14524
14525    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14526        crate::run::MergeRepair {
14527            question: "repair-question".into(),
14528            head: "old-head".into(),
14529            pr: "https://github.com/o/r/pull/42".into(),
14530            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14531            gate: vec![outcome(Some(0), false)],
14532            gate_head: Some("old-head".into()),
14533            gate_fixes: Vec::new(),
14534            pre_gate: Vec::new(),
14535            phase,
14536            failure: None,
14537        }
14538    }
14539
14540    #[tokio::test]
14541    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14542        crate::run::pin_test_home();
14543        let mut runner = runner_at(RunStatus::Landing);
14544        runner.state.merge_repairs.push(merge_repair_request(
14545            crate::run::MergeRepairPhase::Requested,
14546        ));
14547        runner.state.save().unwrap();
14548        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14549        assert!(
14550            runner
14551                .execute()
14552                .await
14553                .unwrap_err()
14554                .to_string()
14555                .contains("already running")
14556        );
14557        assert_eq!(
14558            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14559            crate::run::MergeRepairPhase::Requested
14560        );
14561        drop(claim);
14562        let mut disk = runner.state.clone();
14563        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14564        disk.save().unwrap();
14565        assert!(
14566            runner
14567                .execute()
14568                .await
14569                .unwrap_err()
14570                .to_string()
14571                .contains("decision changed")
14572        );
14573        assert_eq!(
14574            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14575            crate::run::MergeRepairPhase::Complete
14576        );
14577    }
14578
14579    #[test]
14580    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14581        crate::run::pin_test_home();
14582        let mut runner = runner_at(RunStatus::Landing);
14583        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14584        runner.state.reviews = request.reviews.clone();
14585        runner.state.gate = request.gate.clone();
14586        runner.state.gate_ran = true;
14587        runner.state.land_approval = Some(crate::run::LandApproval {
14588            question: "old".into(),
14589            head: "old-head".into(),
14590        });
14591        runner.state.merge_repairs.push(request);
14592        runner.begin_merge_repair_review(0).unwrap();
14593        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14594        assert!(!runner.state.gate_status().ok());
14595        assert!(runner.state.land_approval.is_none());
14596        let disk = RunState::load(&runner.state.id).unwrap();
14597        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14598        assert!(disk.merge_repairs[0].gate[0].ok());
14599        assert_eq!(
14600            disk.merge_repairs[0].phase,
14601            crate::run::MergeRepairPhase::Reviewing
14602        );
14603        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14604        assert!(disk.reviews.is_empty());
14605    }
14606
14607    #[tokio::test]
14608    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14609        crate::run::pin_test_home();
14610        for phase in [
14611            crate::run::MergeRepairPhase::Fixing,
14612            crate::run::MergeRepairPhase::Requested,
14613        ] {
14614            let mut runner = runner_at(RunStatus::Landing);
14615            runner.state.config.graph.land_rounds = 0;
14616            runner.state.merge_repairs.push(merge_repair_request(phase));
14617            runner.state.save().unwrap();
14618            assert!(!runner.repair_merge_review().await.unwrap());
14619            assert_eq!(runner.state.status, RunStatus::Blocked);
14620            assert_eq!(
14621                runner.state.merge_repairs[0].phase,
14622                crate::run::MergeRepairPhase::Failed
14623            );
14624            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14625            assert!(
14626                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14627                    "interrupted"
14628                } else {
14629                    "budget exhausted"
14630                })
14631            );
14632            assert!(runner.state.merge.is_none());
14633        }
14634    }
14635
14636    /// `park_here` folding in the reason `Pause::park_because` recorded -
14637    /// this is what lets an operator reading a run's events tell an
14638    /// interrupt-driven park from an ordinary shutdown park.
14639    #[test]
14640    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14641        crate::run::pin_test_home();
14642        let mut runner = runner_at(RunStatus::Implementing);
14643        let interrupt = Pause::new();
14644        runner.watch_interrupt(interrupt.clone());
14645
14646        interrupt.park_because("task a1b2 asked to run first");
14647
14648        assert!(runner.park_here().expect("park_here"));
14649        assert!(runner.state.parked);
14650        let last = runner.state.events.last().expect("a park event");
14651        assert_eq!(last.node, "park");
14652        assert!(
14653            last.message.contains("task a1b2 asked to run first"),
14654            "expected the interrupt reason in {:?}",
14655            last.message
14656        );
14657    }
14658
14659    /// The cause is stored on the pass when it parks, and the first cause a
14660    /// shared pause was given is the one that stays.
14661    #[test]
14662    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14663        crate::run::pin_test_home();
14664        let mut runner = runner_at(RunStatus::Implementing);
14665        runner.state.begin_pass();
14666        let shutdown = Pause::new();
14667        runner.on_pause(shutdown.clone());
14668        let upgrade = ParkCause::Upgrade {
14669            from: "0.121.0".to_owned(),
14670            to: Some("0.121.1".to_owned()),
14671        };
14672        shutdown.park_for(upgrade.clone());
14673        shutdown.park_for(ParkCause::Operator);
14674
14675        assert!(runner.park_here().expect("park_here"));
14676        assert_eq!(
14677            runner.state.passes[0].end,
14678            Some(crate::run::PassEnd::Parked { cause: upgrade })
14679        );
14680        let last = runner.state.events.last().expect("a park event");
14681        assert!(last.message.contains("0.121.0"), "{}", last.message);
14682    }
14683
14684    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14685    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14686    /// cleared) must not make a *different* run - one only watching its own,
14687    /// unshared interrupt `Pause` - see itself as parked. If a future change
14688    /// ever collapsed these back into one handle, the interrupt scheduler
14689    /// would park every run for the rest of the daemon's life, not just the
14690    /// one it meant to interrupt.
14691    #[test]
14692    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14693        crate::run::pin_test_home();
14694        let mut runner = runner_at(RunStatus::Implementing);
14695        let shutdown = Pause::new();
14696        runner.on_pause(shutdown.clone());
14697        let interrupt = Pause::new();
14698        runner.watch_interrupt(interrupt.clone());
14699
14700        // Nobody has asked for anything yet.
14701        assert!(!runner.park_here().expect("park_here"));
14702        assert!(!runner.state.parked);
14703
14704        // Only the interrupt handle fires; the shutdown handle stays clear.
14705        interrupt.park_because("test");
14706        assert!(!shutdown.parked());
14707        assert!(runner.park_here().expect("park_here"));
14708    }
14709
14710    /// The property every prior attempt at this feature failed to pin down:
14711    /// asking a run to park while one of its nodes has a real, in-flight
14712    /// async operation running (an agent call, in production) must not cut
14713    /// that operation short. `park_here` is only ever consulted *between*
14714    /// `execute`'s node calls - see its own doc - so nothing inside a node
14715    /// can observe a park request until the node itself returns. This proves
14716    /// that structurally, with real `tokio` concurrency and a channel
14717    /// handshake (never a sleep, which would only prove "usually", not
14718    /// "cannot"): the "node" below reports that it has genuinely started,
14719    /// and only then is the park requested; the node still has to be told to
14720    /// finish before `park_here` is ever called, exactly mirroring every
14721    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14722    /// in `execute`.
14723    #[tokio::test]
14724    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14725        crate::run::pin_test_home();
14726        let mut runner = runner_at(RunStatus::Implementing);
14727        let interrupt = Pause::new();
14728        runner.watch_interrupt(interrupt.clone());
14729
14730        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14731        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14732
14733        // Stands in for one node's in-flight agent call: it proves it has
14734        // genuinely started, then blocks - exactly as a spawned CLI process
14735        // does - until told to finish.
14736        let node = async move {
14737            started_tx.send(()).expect("send started");
14738            finish_rx.await.expect("recv finish");
14739            "node finished"
14740        };
14741
14742        let interrupter = async move {
14743            started_rx.await.expect("recv started");
14744            // The call is now genuinely in flight. Ask it to park.
14745            interrupt.park_because("higher-priority task waiting");
14746            // Nothing the node does can observe this yet - there is no
14747            // check inside it, by construction - so let the executor run
14748            // anything pending and then let the node finish on its own.
14749            tokio::task::yield_now().await;
14750            finish_tx.send(()).expect("send finish");
14751        };
14752
14753        let (node_result, ()) = tokio::join!(node, interrupter);
14754        assert_eq!(
14755            node_result, "node finished",
14756            "the in-flight call ran to completion"
14757        );
14758
14759        // Only now, at the boundary the real `execute` would check right
14760        // after this node, does the park take effect.
14761        assert!(runner.park_here().expect("park_here"));
14762        assert!(runner.state.parked);
14763    }
14764
14765    /// A run parked mid-competition carries every field it had accumulated
14766    /// through the exact same disk round-trip an ordinary resume uses -
14767    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
14768    /// Nothing about parking for an interrupt is a special case of that path;
14769    /// this is what proves it rather than assuming it.
14770    #[test]
14771    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14772        crate::run::pin_test_home();
14773        let mut runner = runner_at(RunStatus::Judging);
14774        // `Runner::resume` re-resolves roles from the saved config, which
14775        // refuses an empty roster - give it the same minimal one `conductor`
14776        // itself uses.
14777        runner.state.config.agents = vec![conductor()];
14778        runner.state.candidates = vec![Candidate {
14779            index: 0,
14780            label: 'A',
14781            agent: "alpha".to_owned(),
14782            branch: "magi/x/A".to_owned(),
14783            worktree: PathBuf::from("/nonexistent/worktree"),
14784            summary: "did the thing".to_owned(),
14785            stat: "1 file changed".to_owned(),
14786            files: 1,
14787            commits: 1,
14788            empty: false,
14789            failed: None,
14790            verified_noop: None,
14791            duration_ms: 1234,
14792            folded: false,
14793        }];
14794        let run_id = runner.state.id.clone();
14795
14796        let interrupt = Pause::new();
14797        runner.watch_interrupt(interrupt.clone());
14798        interrupt.park_because("task c3d4 asked to run first");
14799        assert!(runner.park_here().expect("park_here"));
14800
14801        let resumed = Runner::resume(&run_id).expect("resume");
14802        assert_eq!(resumed.state.candidates.len(), 1);
14803        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14804        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14805        assert_eq!(resumed.state.status, runner.state.status);
14806        assert!(
14807            resumed.state.parked,
14808            "still parked until `execute` actually walks the graph again"
14809        );
14810        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14811    }
14812
14813    /// A fresh open question on `run`, stored and handed back for assertions.
14814    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14815        let mut q = ask::Question::new(
14816            run.to_owned(),
14817            "implement".to_owned(),
14818            "impl-A".to_owned(),
14819            "Which storage backend should the cache use?".to_owned(),
14820            String::new(),
14821            vec!["SQLite".to_owned(), "Redis".to_owned()],
14822        );
14823        store.put(&mut q).unwrap();
14824        q
14825    }
14826
14827    #[test]
14828    fn a_failed_runs_open_question_is_abandoned() {
14829        ask_test_home();
14830        let store = ask::Questions::open();
14831        let mut runner = runner_at(RunStatus::Failed);
14832        let run = runner.state.id.clone();
14833        let q = ask_open_question(&store, &run);
14834
14835        runner.settle_questions();
14836
14837        let back = store.get(&q.id).unwrap();
14838        assert!(
14839            !back.status.open(),
14840            "the seat that asked died with the run; nobody is left to read an answer"
14841        );
14842        assert!(
14843            back.detail.contains(&run) && back.detail.contains("failed"),
14844            "the reason names what the run became, not just that it is gone: {}",
14845            back.detail
14846        );
14847    }
14848
14849    #[test]
14850    fn a_merged_runs_open_question_is_abandoned_too() {
14851        ask_test_home();
14852        let store = ask::Questions::open();
14853        // A run that finishes cleanly still leaves nobody to read an answer -
14854        // this is not only a failure-path cleanup.
14855        for status in [RunStatus::Merged, RunStatus::Ready] {
14856            let mut runner = runner_at(status);
14857            let run = runner.state.id.clone();
14858            let q = ask_open_question(&store, &run);
14859
14860            runner.settle_questions();
14861
14862            let back = store.get(&q.id).unwrap();
14863            assert!(
14864                !back.status.open(),
14865                "{status:?} run's question must not outlive the run"
14866            );
14867        }
14868    }
14869
14870    #[test]
14871    fn a_still_resumable_runs_open_question_is_left_alone() {
14872        ask_test_home();
14873        let store = ask::Questions::open();
14874        // `Blocked` and `Stalled` can still be resumed — the candidates, the
14875        // review round and the seat sessions are all still on disk — so a
14876        // question asked mid-round may yet get a real answer from a real
14877        // resume. Sweeping it here would be exactly the failure mode this
14878        // whole feature exists to avoid on the other side.
14879        for status in [RunStatus::Blocked, RunStatus::Stalled] {
14880            let mut runner = runner_at(status);
14881            let run = runner.state.id.clone();
14882            let q = ask_open_question(&store, &run);
14883
14884            runner.settle_questions();
14885
14886            let back = store.get(&q.id).unwrap();
14887            assert!(
14888                back.status.open(),
14889                "{status:?} is still alive; the question must still be waiting"
14890            );
14891        }
14892    }
14893
14894    #[test]
14895    fn settle_questions_never_touches_an_already_answered_question() {
14896        ask_test_home();
14897        let store = ask::Questions::open();
14898        let mut runner = runner_at(RunStatus::Failed);
14899        let run = runner.state.id.clone();
14900        let mut q = ask_open_question(&store, &run);
14901        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14902            .unwrap();
14903        store.put(&mut q).unwrap();
14904
14905        // Called twice, the way a crash-recovered daemon reclaim and the
14906        // graph's own cleanup both can for the same run — `abandon_for_run`
14907        // only ever touches what is still open, so this must be inert both
14908        // times, not merely the second.
14909        runner.settle_questions();
14910        runner.settle_questions();
14911
14912        let back = store.get(&q.id).unwrap();
14913        assert_eq!(
14914            back.status,
14915            ask::QuestionStatus::Answered,
14916            "a real answer is a decision on record, never overwritten by a sweep"
14917        );
14918    }
14919
14920    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
14921    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
14922    /// without merging, whose winner is still the operator's answer to read.
14923    /// Nothing previously called `fold_run` itself with a real `tally`, so
14924    /// this is the first test to pin down the one distinction the whole
14925    /// automatic-fold feature depends on: the winner's worktree and branch
14926    /// must survive, everything else sharing the run's worktree bay - a
14927    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
14928    /// second sweep treats every non-winner directory under the bay alike -
14929    /// must not.
14930    #[tokio::test]
14931    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14932        crate::run::pin_test_home();
14933        let tmp = tempfile::tempdir().expect("tempdir");
14934        let repo = tmp.path().join("repo");
14935        std::fs::create_dir_all(&repo).unwrap();
14936        init_repo(&repo);
14937
14938        let mut config = Config::default();
14939        config.graph.worktree_root = Some(tmp.path().join("wt"));
14940
14941        let mut state = RunState::new(
14942            repo.clone(),
14943            "main".to_owned(),
14944            "deadbeef".to_owned(),
14945            "task".to_owned(),
14946            config,
14947        );
14948        let root = state.worktree_root();
14949        let wt_a = root.join("cand-A");
14950        let wt_b = root.join("cand-B");
14951        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14952            .await
14953            .expect("worktree A");
14954        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14955            .await
14956            .expect("worktree B");
14957
14958        state.candidates = vec![
14959            Candidate {
14960                index: 0,
14961                label: 'A',
14962                agent: "alpha".to_owned(),
14963                branch: "magi/x/A".to_owned(),
14964                worktree: wt_a.clone(),
14965                summary: String::new(),
14966                stat: String::new(),
14967                files: 0,
14968                commits: 0,
14969                empty: false,
14970                failed: None,
14971                verified_noop: None,
14972                duration_ms: 0,
14973                folded: false,
14974            },
14975            Candidate {
14976                index: 1,
14977                label: 'B',
14978                agent: "beta".to_owned(),
14979                branch: "magi/x/B".to_owned(),
14980                worktree: wt_b.clone(),
14981                summary: String::new(),
14982                stat: String::new(),
14983                files: 0,
14984                commits: 0,
14985                empty: false,
14986                failed: None,
14987                verified_noop: None,
14988                duration_ms: 0,
14989                folded: false,
14990            },
14991        ];
14992        state.tally = Some(Tally {
14993            first_choice: BTreeMap::from([('A', 1)]),
14994            borda: BTreeMap::new(),
14995            winner: 'A',
14996            rankings: 1,
14997            unanimous_initial: true,
14998            deliberated: false,
14999            changed_votes: 0,
15000            unanimous_final: true,
15001            tie_break: None,
15002            judges: 1,
15003            present: 1,
15004            quorum: 1,
15005            met_quorum: true,
15006            uncontested: None,
15007        });
15008        state.status = RunStatus::Ready;
15009
15010        fold_run(&mut state, false, &crate::run::home())
15011            .await
15012            .expect("fold_run");
15013
15014        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15015        assert!(
15016            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15017            "the unmerged winner's branch survives"
15018        );
15019        assert!(
15020            !state.candidates[0].folded,
15021            "the winner is not marked folded"
15022        );
15023
15024        assert!(!wt_b.exists(), "the loser's worktree is removed");
15025        assert!(
15026            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15027            "the loser's branch is removed"
15028        );
15029        assert!(state.candidates[1].folded, "the loser is marked folded");
15030    }
15031
15032    #[tokio::test]
15033    #[cfg(unix)]
15034    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15035        use std::os::fd::AsRawFd as _;
15036        use std::time::Duration;
15037
15038        let tmp = tempfile::tempdir().unwrap();
15039        let repo = tmp.path().join("repo");
15040        std::fs::create_dir_all(&repo).unwrap();
15041        init_repo(&repo);
15042        let common = repo.join(".git");
15043        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15044        let home = tmp.path().join("home");
15045        let mut config = Config::default();
15046        config.coordination = crate::config::test_coordination();
15047        config.graph.worktree_root = Some(tmp.path().join("wt"));
15048        config.graph.timeout_resource_wait = 1;
15049        let mut state = RunState::new(
15050            repo.clone(),
15051            "main".into(),
15052            "deadbeef".into(),
15053            "task".into(),
15054            config,
15055        );
15056        let root = state.worktree_root();
15057        let candidate = root.join("cand-A");
15058        let reviewer = root.join("review-1");
15059        let branch = state.branch_for('A');
15060        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15061            .await
15062            .unwrap();
15063        git::worktree_add_detached(&repo, &reviewer, "main")
15064            .await
15065            .unwrap();
15066        git::acquire_worktree_config(&repo).await.unwrap();
15067        state.enabled_worktree_config = true;
15068        state.status = RunStatus::Ready;
15069        state.candidates.push(Candidate {
15070            index: 0,
15071            label: 'A',
15072            agent: "implement".into(),
15073            branch: branch.clone(),
15074            worktree: candidate.clone(),
15075            summary: String::new(),
15076            stat: String::new(),
15077            files: 0,
15078            commits: 0,
15079            empty: false,
15080            failed: None,
15081            verified_noop: None,
15082            duration_ms: 0,
15083            folded: false,
15084        });
15085        state.save_under(&home).unwrap();
15086        let config_before = std::fs::read(common.join("config")).unwrap();
15087        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15088            .await
15089            .unwrap();
15090        // A separate OS handle bypasses the launcher's local gate, exactly as
15091        // an external updater does, and must still prevent all folding work.
15092        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15093        assert_eq!(
15094            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15095            0
15096        );
15097        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15098        assert!(
15099            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15100                .await
15101                .is_err()
15102        );
15103        assert!(candidate.exists());
15104        assert!(reviewer.exists());
15105        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15106        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15107        assert_eq!(
15108            git::git(&repo, &["worktree", "list", "--porcelain"])
15109                .await
15110                .unwrap(),
15111            worktrees_before
15112        );
15113        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15114            .await
15115            .unwrap()
15116            .unwrap_err();
15117        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15118        drop(fold);
15119        assert!(candidate.exists() && reviewer.exists());
15120        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15121        assert_eq!(
15122            state.resource_block.as_ref().unwrap().kind,
15123            "repository coordination"
15124        );
15125        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15126        drop(external);
15127        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15128        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15129            .await
15130            .unwrap()
15131            .unwrap();
15132        drop(fold);
15133        assert!(removed.contains(&branch));
15134        assert!(!root.exists());
15135        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15136        assert!(state.candidates[0].folded);
15137        assert!(!state.enabled_worktree_config);
15138        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15139        let enabled = git::git_raw(
15140            &repo,
15141            &["config", "--local", "--get", "extensions.worktreeConfig"],
15142        )
15143        .await
15144        .unwrap();
15145        assert!(!enabled.ok(), "the last config reference was released");
15146    }
15147
15148    /// A branch handed to a later run is that run's (and its pull request's):
15149    /// folding the run that released it must not delete it.
15150    #[tokio::test]
15151    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15152        let tmp = tempfile::tempdir().expect("tempdir");
15153        let repo = tmp.path().join("repo");
15154        std::fs::create_dir_all(&repo).unwrap();
15155        init_repo(&repo);
15156        let home = tmp.path().join("home");
15157
15158        let mut config = Config::default();
15159        config.graph.worktree_root = Some(tmp.path().join("wt"));
15160        let mut state = RunState::new(
15161            repo.clone(),
15162            "main".to_owned(),
15163            "deadbeef".to_owned(),
15164            "task".to_owned(),
15165            config,
15166        );
15167        // The worktree is already gone (released); the branch survives.
15168        git::git(&repo, &["branch", "magi/x/A", "main"])
15169            .await
15170            .expect("branch");
15171        state.candidates = vec![Candidate {
15172            index: 0,
15173            label: 'A',
15174            agent: "alpha".to_owned(),
15175            branch: "magi/x/A".to_owned(),
15176            worktree: state.worktree_root().join("cand-A"),
15177            summary: String::new(),
15178            stat: String::new(),
15179            files: 0,
15180            commits: 0,
15181            empty: false,
15182            failed: None,
15183            verified_noop: None,
15184            duration_ms: 0,
15185            folded: true,
15186        }];
15187        state.released_to = Some("20260901-000000-new1".to_owned());
15188        state.released_branches = vec!["magi/x/A".to_owned()];
15189
15190        fold_run(&mut state, true, &home).await.expect("fold_run");
15191
15192        assert!(
15193            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15194            "the handed-over branch survives a fold"
15195        );
15196    }
15197
15198    /// A winner with nothing ahead of the base is caught before `gh` is ever
15199    /// asked for a pull request, and the message carries what the task's
15200    /// references resolved to.
15201    #[tokio::test]
15202    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15203        let tmp = tempfile::tempdir().expect("tempdir");
15204        let repo = tmp.path().join("repo");
15205        std::fs::create_dir_all(&repo).unwrap();
15206        init_repo(&repo);
15207        let run = |args: &[&str]| {
15208            let out = std::process::Command::new("git")
15209                .quiet()
15210                .args(args)
15211                .current_dir(&repo)
15212                .output()
15213                .expect("spawn git");
15214            assert!(out.status.success(), "git {args:?}");
15215        };
15216        run(&["branch", "magi/x/A"]);
15217        run(&["checkout", "-q", "-b", "magi/x/B"]);
15218        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15219        run(&["add", "-A"]);
15220        run(&["commit", "-q", "-m", "work"]);
15221        run(&["checkout", "-q", "main"]);
15222
15223        // A pull request is compared against the remote's base, so the
15224        // fixture needs one. Before it exists nothing can be read, and the
15225        // answer must be "not empty".
15226        let probe = RunState::new(
15227            repo.clone(),
15228            "main".to_owned(),
15229            "deadbeef".to_owned(),
15230            "task".to_owned(),
15231            Config::default(),
15232        );
15233        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15234        let bare = tmp.path().join("origin.git");
15235        let out = std::process::Command::new("git")
15236            .quiet()
15237            .args(["init", "-q", "--bare"])
15238            .arg(&bare)
15239            .output()
15240            .expect("spawn git");
15241        assert!(out.status.success(), "git init --bare");
15242        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15243        run(&["push", "-q", "origin", "main"]);
15244
15245        let mut state = RunState::new(
15246            repo.clone(),
15247            "main".to_owned(),
15248            "deadbeef".to_owned(),
15249            "task".to_owned(),
15250            Config::default(),
15251        );
15252        state.seeds = vec![refs::Seed {
15253            token: "magi/27b2/A".to_owned(),
15254            kind: refs::SeedKind::Unresolved,
15255            sha: String::new(),
15256            branch: true,
15257            detail: "no branch or commit named magi/27b2/A".to_owned(),
15258        }];
15259
15260        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15261        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15262        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15263        let detail = empty_candidate_detail(&state, "main");
15264        assert!(detail.starts_with("empty candidate"), "{detail}");
15265        assert!(detail.contains("magi/27b2/A"), "{detail}");
15266    }
15267
15268    /// `status == Ready` used to be read as "this is the harmless
15269    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15270    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15271    /// run's PR was closed without merging — and reentering `merge` with
15272    /// `mode` still `Pr` does not know the difference, so it pushed and
15273    /// opened a second pull request. `mode == Local` reproduces the same
15274    /// blind spot without a network call: reentry must not attempt another
15275    /// git merge once this node has already recorded an outcome.
15276    #[tokio::test]
15277    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15278        let tmp = tempfile::tempdir().expect("tempdir");
15279        let repo = tmp.path().join("repo");
15280        std::fs::create_dir_all(&repo).unwrap();
15281        init_repo(&repo);
15282
15283        let mut config = Config::default();
15284        config.merge.mode = MergeMode::Local;
15285
15286        let mut state = RunState::new(
15287            repo.clone(),
15288            "main".to_owned(),
15289            "deadbeef".to_owned(),
15290            "task".to_owned(),
15291            config,
15292        );
15293        state.candidates = vec![Candidate {
15294            index: 0,
15295            label: 'A',
15296            agent: "alpha".to_owned(),
15297            branch: "does-not-exist".to_owned(),
15298            worktree: repo.clone(),
15299            summary: String::new(),
15300            stat: String::new(),
15301            files: 0,
15302            commits: 0,
15303            empty: false,
15304            failed: None,
15305            verified_noop: None,
15306            duration_ms: 0,
15307            folded: false,
15308        }];
15309        state.tally = Some(Tally {
15310            first_choice: BTreeMap::from([('A', 1)]),
15311            borda: BTreeMap::new(),
15312            winner: 'A',
15313            rankings: 1,
15314            unanimous_initial: true,
15315            deliberated: false,
15316            changed_votes: 0,
15317            unanimous_final: true,
15318            tie_break: None,
15319            judges: 0,
15320            present: 0,
15321            quorum: 0,
15322            met_quorum: true,
15323            uncontested: Some("only candidate A produced a change".to_owned()),
15324        });
15325        state.reviews = vec![ReviewRound {
15326            round: 1,
15327            head: "deadbeef".to_owned(),
15328            verified_head: None,
15329            verified_at: None,
15330            reviews: Vec::new(),
15331            e2e: Vec::new(),
15332            fix: None,
15333            blocking: 0,
15334            answered: 0,
15335            expected: 0,
15336            clean: true,
15337            verify_retried: false,
15338            e2e_deferred: false,
15339            e2e_defer_reason: None,
15340            progressed: false,
15341            vote_split: false,
15342            reconsideration: Vec::new(),
15343            verdict: None,
15344        }];
15345        state.gate = vec![CommandOutcome {
15346            command: "test".to_owned(),
15347            code: Some(0),
15348            output_tail: String::new(),
15349            duration_ms: 0,
15350            resource_blocked: false,
15351        }];
15352        state.gate_ran = true;
15353        // Reached its conclusion already — e.g. `land` closing the PR without
15354        // merging it, which (like the honest `MergeMode::None` path) leaves
15355        // `status` at `Ready`. The recorded outcome is what actually marks
15356        // this node done.
15357        state.status = RunStatus::Ready;
15358        state.merge = Some(MergeOutcome {
15359            mode: MergeMode::Local,
15360            ok: false,
15361            detail: "already concluded".to_owned(),
15362            empty: false,
15363        });
15364
15365        let mut runner = Runner {
15366            state,
15367            roles: ResolvedRoles {
15368                implementers: Vec::new(),
15369                judges: Vec::new(),
15370                reviewers: Vec::new(),
15371                fixer: None,
15372                conductor: conductor(),
15373                implementer_roster: Vec::new(),
15374                judge_roster: Vec::new(),
15375                reviewer_roster: Vec::new(),
15376            },
15377            sem: Arc::new(Semaphore::new(1)),
15378            pause: Pause::new(),
15379            interrupt: Pause::new(),
15380        };
15381
15382        runner.merge().await.expect("merge");
15383
15384        assert_eq!(
15385            runner.state.status,
15386            RunStatus::Ready,
15387            "a concluded run's status must not change on reentry"
15388        );
15389        assert_eq!(
15390            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15391            Some("already concluded"),
15392            "merge must not run again once the node already recorded an outcome"
15393        );
15394    }
15395
15396    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15397    /// when its last attempt was resource-blocked (the shared build cache
15398    /// could not be acquired or confirmed fresh in time - see
15399    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15400    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15401    /// a run merge a tree the gate never actually checked - exactly the case
15402    /// a contended cache produces on every retry until it clears. `merge`
15403    /// must refuse until `gate` has actually recorded an attempt.
15404    #[tokio::test]
15405    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15406        let tmp = tempfile::tempdir().expect("tempdir");
15407        let repo = tmp.path().join("repo");
15408        std::fs::create_dir_all(&repo).unwrap();
15409        init_repo(&repo);
15410
15411        let mut config = Config::default();
15412        config.merge.mode = MergeMode::Local;
15413
15414        let mut state = RunState::new(
15415            repo.clone(),
15416            "main".to_owned(),
15417            "deadbeef".to_owned(),
15418            "task".to_owned(),
15419            config,
15420        );
15421        state.candidates = vec![Candidate {
15422            index: 0,
15423            label: 'A',
15424            agent: "alpha".to_owned(),
15425            branch: "does-not-exist".to_owned(),
15426            worktree: repo.clone(),
15427            summary: String::new(),
15428            stat: String::new(),
15429            files: 0,
15430            commits: 0,
15431            empty: false,
15432            failed: None,
15433            verified_noop: None,
15434            duration_ms: 0,
15435            folded: false,
15436        }];
15437        state.tally = Some(Tally {
15438            first_choice: BTreeMap::from([('A', 1)]),
15439            borda: BTreeMap::new(),
15440            winner: 'A',
15441            rankings: 1,
15442            unanimous_initial: true,
15443            deliberated: false,
15444            changed_votes: 0,
15445            unanimous_final: true,
15446            tie_break: None,
15447            judges: 0,
15448            present: 0,
15449            quorum: 0,
15450            met_quorum: true,
15451            uncontested: Some("only candidate A produced a change".to_owned()),
15452        });
15453        state.reviews = vec![ReviewRound {
15454            round: 1,
15455            head: "deadbeef".to_owned(),
15456            verified_head: None,
15457            verified_at: None,
15458            reviews: Vec::new(),
15459            e2e: Vec::new(),
15460            fix: None,
15461            blocking: 0,
15462            answered: 0,
15463            expected: 0,
15464            clean: true,
15465            verify_retried: false,
15466            e2e_deferred: false,
15467            e2e_defer_reason: None,
15468            progressed: false,
15469            vote_split: false,
15470            reconsideration: Vec::new(),
15471            verdict: None,
15472        }];
15473        // The point: `gate` has not recorded anything yet.
15474        state.gate = Vec::new();
15475        state.gate_ran = false;
15476        state.status = RunStatus::Gating;
15477
15478        let mut runner = Runner {
15479            state,
15480            roles: ResolvedRoles {
15481                implementers: Vec::new(),
15482                judges: Vec::new(),
15483                reviewers: Vec::new(),
15484                fixer: None,
15485                conductor: conductor(),
15486                implementer_roster: Vec::new(),
15487                judge_roster: Vec::new(),
15488                reviewer_roster: Vec::new(),
15489            },
15490            sem: Arc::new(Semaphore::new(1)),
15491            pause: Pause::new(),
15492            interrupt: Pause::new(),
15493        };
15494
15495        runner.merge().await.expect("merge");
15496
15497        assert!(
15498            runner.state.merge.is_none(),
15499            "an empty gate must never be read as a passing one: {:?}",
15500            runner.state.merge
15501        );
15502    }
15503
15504    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
15505    /// commands configured and `merge.mode` is `none` (a review-only run).
15506    /// `gate` must still record a real attempt — zero commands, vacuously
15507    /// passed — rather than leaving `state.gate` empty in a way `merge`
15508    /// cannot tell apart from "never ran"; otherwise the run reaches
15509    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
15510    #[tokio::test]
15511    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15512        let tmp = tempfile::tempdir().expect("tempdir");
15513        let repo = tmp.path().join("repo");
15514        std::fs::create_dir_all(&repo).unwrap();
15515        init_repo(&repo);
15516
15517        // Default config: `verify.gate` empty, `merge.mode` is `none`.
15518        let config = Config::default();
15519
15520        let mut state = RunState::new(
15521            repo.clone(),
15522            "main".to_owned(),
15523            "deadbeef".to_owned(),
15524            "task".to_owned(),
15525            config,
15526        );
15527        state.candidates = vec![Candidate {
15528            index: 0,
15529            label: 'A',
15530            agent: "alpha".to_owned(),
15531            branch: "does-not-exist".to_owned(),
15532            worktree: repo.clone(),
15533            summary: String::new(),
15534            stat: String::new(),
15535            files: 0,
15536            commits: 0,
15537            empty: false,
15538            failed: None,
15539            verified_noop: None,
15540            duration_ms: 0,
15541            folded: false,
15542        }];
15543        state.tally = Some(Tally {
15544            first_choice: BTreeMap::from([('A', 1)]),
15545            borda: BTreeMap::new(),
15546            winner: 'A',
15547            rankings: 1,
15548            unanimous_initial: true,
15549            deliberated: false,
15550            changed_votes: 0,
15551            unanimous_final: true,
15552            tie_break: None,
15553            judges: 0,
15554            present: 0,
15555            quorum: 0,
15556            met_quorum: true,
15557            uncontested: Some("only candidate A produced a change".to_owned()),
15558        });
15559        state.reviews = vec![ReviewRound {
15560            round: 1,
15561            head: "deadbeef".to_owned(),
15562            verified_head: None,
15563            verified_at: None,
15564            reviews: Vec::new(),
15565            e2e: Vec::new(),
15566            fix: None,
15567            blocking: 0,
15568            answered: 0,
15569            expected: 0,
15570            clean: true,
15571            verify_retried: false,
15572            e2e_deferred: false,
15573            e2e_defer_reason: None,
15574            progressed: false,
15575            vote_split: false,
15576            reconsideration: Vec::new(),
15577            verdict: None,
15578        }];
15579
15580        let mut runner = Runner {
15581            state,
15582            roles: ResolvedRoles {
15583                implementers: Vec::new(),
15584                judges: Vec::new(),
15585                reviewers: Vec::new(),
15586                fixer: None,
15587                conductor: conductor(),
15588                implementer_roster: Vec::new(),
15589                judge_roster: Vec::new(),
15590                reviewer_roster: Vec::new(),
15591            },
15592            sem: Arc::new(Semaphore::new(1)),
15593            pause: Pause::new(),
15594            interrupt: Pause::new(),
15595        };
15596
15597        runner.gate().await.expect("gate");
15598        assert!(
15599            runner.state.gate_ran,
15600            "zero configured commands is still a real attempt, not an unrun gate"
15601        );
15602        assert!(runner.state.gate.is_empty());
15603        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15604        assert_ne!(
15605            runner.state.status,
15606            RunStatus::Blocked,
15607            "a gate with nothing to check must not read as failed"
15608        );
15609
15610        runner.merge().await.expect("merge");
15611        assert_eq!(
15612            runner.state.status,
15613            RunStatus::Ready,
15614            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15615        );
15616    }
15617
15618    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15619    /// `verify.gate` (so the e2e leg and the final gate never build against
15620    /// different directories). With zero `verify.gate` commands but a
15621    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15622    /// that lease before discovering it had nothing to run - so a repo with
15623    /// no gate commands could come back `resource_blocked` (and therefore
15624    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15625    /// cache, exactly the contention this run's own zero commands could
15626    /// never have touched. `gate` must recognise there is nothing to check
15627    /// before it ever asks for the lease.
15628    #[tokio::test]
15629    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15630        crate::run::pin_test_home();
15631        let home = crate::run::home();
15632
15633        let tmp = tempfile::tempdir().expect("tempdir");
15634        let repo = tmp.path().join("repo");
15635        std::fs::create_dir_all(&repo).unwrap();
15636        init_repo(&repo);
15637        // Unique to this test, so holding its lease cannot collide with
15638        // another test sharing the same process-wide `home`.
15639        let cache_dir = tmp.path().join("target");
15640
15641        let mut config = Config::default();
15642        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15643        // `verify.gate` stays empty (the default). Bounded so a regression
15644        // that does start waiting fails the test in seconds, not hangs it.
15645        config.graph.timeout_verify = Some(2);
15646        config.graph.timeout_resource_wait = 2;
15647
15648        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15649        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15650            .expect("no io error acquiring directly")
15651        {
15652            crate::cache::AcquireOutcome::Acquired(g) => g,
15653            crate::cache::AcquireOutcome::Busy(b) => {
15654                panic!("expected the direct acquire to win the lease first: {b:?}")
15655            }
15656        };
15657
15658        let mut state = RunState::new(
15659            repo.clone(),
15660            "main".to_owned(),
15661            "deadbeef".to_owned(),
15662            "task".to_owned(),
15663            config,
15664        );
15665        state.candidates = vec![Candidate {
15666            index: 0,
15667            label: 'A',
15668            agent: "alpha".to_owned(),
15669            branch: "does-not-exist".to_owned(),
15670            worktree: repo.clone(),
15671            summary: String::new(),
15672            stat: String::new(),
15673            files: 0,
15674            commits: 0,
15675            empty: false,
15676            failed: None,
15677            verified_noop: None,
15678            duration_ms: 0,
15679            folded: false,
15680        }];
15681        state.tally = Some(Tally {
15682            first_choice: BTreeMap::from([('A', 1)]),
15683            borda: BTreeMap::new(),
15684            winner: 'A',
15685            rankings: 1,
15686            unanimous_initial: true,
15687            deliberated: false,
15688            changed_votes: 0,
15689            unanimous_final: true,
15690            tie_break: None,
15691            judges: 0,
15692            present: 0,
15693            quorum: 0,
15694            met_quorum: true,
15695            uncontested: Some("only candidate A produced a change".to_owned()),
15696        });
15697        state.reviews = vec![ReviewRound {
15698            round: 1,
15699            head: "deadbeef".to_owned(),
15700            verified_head: None,
15701            verified_at: None,
15702            reviews: Vec::new(),
15703            e2e: Vec::new(),
15704            fix: None,
15705            blocking: 0,
15706            answered: 0,
15707            expected: 0,
15708            clean: true,
15709            verify_retried: false,
15710            e2e_deferred: false,
15711            e2e_defer_reason: None,
15712            progressed: false,
15713            vote_split: false,
15714            reconsideration: Vec::new(),
15715            verdict: None,
15716        }];
15717
15718        let mut runner = Runner {
15719            state,
15720            roles: ResolvedRoles {
15721                implementers: Vec::new(),
15722                judges: Vec::new(),
15723                reviewers: Vec::new(),
15724                fixer: None,
15725                conductor: conductor(),
15726                implementer_roster: Vec::new(),
15727                judge_roster: Vec::new(),
15728                reviewer_roster: Vec::new(),
15729            },
15730            sem: Arc::new(Semaphore::new(1)),
15731            pause: Pause::new(),
15732            interrupt: Pause::new(),
15733        };
15734
15735        let started = std::time::Instant::now();
15736        runner.gate().await.expect("gate");
15737        assert!(
15738            started.elapsed() < Duration::from_secs(1),
15739            "a gate with nothing to run must never wait on a lease it never needed"
15740        );
15741        assert!(
15742            runner.state.gate_ran,
15743            "zero commands is still a real, immediate attempt"
15744        );
15745        assert!(runner.state.gate.is_empty());
15746        assert_ne!(
15747            runner.state.status,
15748            RunStatus::Blocked,
15749            "must not read as resource-blocked on a lease it never asked for"
15750        );
15751    }
15752
15753    #[cfg(unix)]
15754    #[tokio::test]
15755    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15756        crate::run::pin_test_home();
15757        let tmp = tempfile::tempdir().unwrap();
15758        let mut config = Config::default();
15759        config.disk.min_free_bytes = 0;
15760        let mut state = RunState::new(
15761            tmp.path().to_owned(),
15762            "main".into(),
15763            "base".into(),
15764            "verification evidence".into(),
15765            config,
15766        );
15767        let (outcomes, pids) = run_commands(
15768            &mut state,
15769            "gate",
15770            "gate",
15771            1,
15772            &["sh".into(), "-c".into()],
15773            &[
15774                "printf before-timeout; exec sleep 30".into(),
15775                "kill -TERM $$".into(),
15776                "exit 7".into(),
15777                "printf success".into(),
15778            ],
15779            tmp.path(),
15780            Duration::from_secs(1),
15781        )
15782        .await;
15783        wait_for_timed_out_children_to_die(&pids).await;
15784        assert_eq!(
15785            pids.len(),
15786            1,
15787            "unleased timeout retains its death-check PID"
15788        );
15789        assert_eq!(outcomes.len(), 4);
15790        assert_eq!(outcomes[0].code, None);
15791        assert!(outcomes[0].output_tail.contains("before-timeout"));
15792        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15793        assert_eq!(outcomes[1].code, None);
15794        assert!(outcomes[1].output_tail.contains("command terminated:"));
15795        assert!(!outcomes[1].output_tail.contains("timed out"));
15796        assert_eq!(outcomes[2].code, Some(7));
15797        assert!(!outcomes[2].output_tail.contains("timed out"));
15798        assert_eq!(outcomes[3].code, Some(0));
15799        assert_eq!(outcomes[3].output_tail, "success");
15800        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15801    }
15802
15803    /// The addendum's second gap: a `verify.gate` command running for real
15804    /// wall-clock time had nothing at all to show for it in `active` before
15805    /// `run_commands` learned to record it — a run could sit in `Gating` for
15806    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
15807    /// what was actually happening. Proven with a genuinely still-running
15808    /// command, not just a before/after check on the final state: a poller
15809    /// task reads the same `run.json` `gate()` is writing, the same way the
15810    /// phone or `magi show` would, while the shell command is still blocked
15811    /// on its own release marker.
15812    #[tokio::test]
15813    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15814        crate::run::pin_test_home();
15815
15816        let tmp = tempfile::tempdir().expect("tempdir");
15817        let repo = tmp.path().join("repo");
15818        std::fs::create_dir_all(&repo).unwrap();
15819        init_repo(&repo);
15820
15821        let mut config = Config::default();
15822        config.verify.gate = vec![
15823            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15824             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15825                .to_owned(),
15826        ];
15827
15828        let mut state = RunState::new(
15829            repo.clone(),
15830            "main".to_owned(),
15831            "deadbeef".to_owned(),
15832            "task".to_owned(),
15833            config,
15834        );
15835        let run_id = state.id.clone();
15836        state.candidates = vec![Candidate {
15837            index: 0,
15838            label: 'A',
15839            agent: "alpha".to_owned(),
15840            branch: "does-not-exist".to_owned(),
15841            worktree: repo.clone(),
15842            summary: String::new(),
15843            stat: String::new(),
15844            files: 0,
15845            commits: 0,
15846            empty: false,
15847            failed: None,
15848            verified_noop: None,
15849            duration_ms: 0,
15850            folded: false,
15851        }];
15852        state.tally = Some(Tally {
15853            first_choice: BTreeMap::from([('A', 1)]),
15854            borda: BTreeMap::new(),
15855            winner: 'A',
15856            rankings: 1,
15857            unanimous_initial: true,
15858            deliberated: false,
15859            changed_votes: 0,
15860            unanimous_final: true,
15861            tie_break: None,
15862            judges: 0,
15863            present: 0,
15864            quorum: 0,
15865            met_quorum: true,
15866            uncontested: Some("only candidate A produced a change".to_owned()),
15867        });
15868        state.reviews = vec![ReviewRound {
15869            round: 1,
15870            head: "deadbeef".to_owned(),
15871            verified_head: None,
15872            verified_at: None,
15873            reviews: Vec::new(),
15874            e2e: Vec::new(),
15875            fix: None,
15876            blocking: 0,
15877            answered: 0,
15878            expected: 0,
15879            clean: true,
15880            verify_retried: false,
15881            e2e_deferred: false,
15882            e2e_defer_reason: None,
15883            progressed: false,
15884            vote_split: false,
15885            reconsideration: Vec::new(),
15886            verdict: None,
15887        }];
15888
15889        let mut runner = Runner {
15890            state,
15891            roles: ResolvedRoles {
15892                implementers: Vec::new(),
15893                judges: Vec::new(),
15894                reviewers: Vec::new(),
15895                fixer: None,
15896                conductor: conductor(),
15897                implementer_roster: Vec::new(),
15898                judge_roster: Vec::new(),
15899                reviewer_roster: Vec::new(),
15900            },
15901            sem: Arc::new(Semaphore::new(1)),
15902            pause: Pause::new(),
15903            interrupt: Pause::new(),
15904        };
15905
15906        let started_marker = repo.join("started.marker");
15907        let release_marker = repo.join("release.marker");
15908        let poller = tokio::spawn(async move {
15909            // Bounded so a regression that never records the task entry
15910            // fails this test in seconds instead of hanging the suite —
15911            // the same shape `a_park_requested_while_a_seat_is_mid_call_
15912            // does_not_cut_it_short` uses for the same reason.
15913            for _ in 0..100 {
15914                if started_marker.exists()
15915                    && let Ok(s) = crate::run::RunState::load(&run_id)
15916                    && let Some(a) = s.active.get("gate")
15917                {
15918                    std::fs::write(&release_marker, b"go").expect("release marker");
15919                    return Some(a.clone());
15920                }
15921                tokio::time::sleep(Duration::from_millis(50)).await;
15922            }
15923            None
15924        });
15925
15926        runner.gate().await.expect("gate");
15927        let captured = poller.await.expect("poller task");
15928        let captured = captured.expect(
15929            "the poller never saw a `gate` task entry in run.json while the command was \
15930             still blocked on its own release marker",
15931        );
15932
15933        assert_eq!(captured.task.as_deref(), Some("gate"));
15934        assert_eq!(captured.node, "gate");
15935        assert_eq!(captured.index, Some(1));
15936        assert_eq!(captured.total, Some(1));
15937        assert!(
15938            captured
15939                .command
15940                .as_deref()
15941                .is_some_and(|c| c.contains("started.marker")),
15942            "{captured:?}"
15943        );
15944
15945        assert!(
15946            runner.state.active.is_empty(),
15947            "the entry must be cleared once the command actually finished: {:?}",
15948            runner.state.active
15949        );
15950        assert!(runner.state.gate_ran);
15951        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15952    }
15953
15954    /// The hand-off over a blocking finding a reviewer rejected on leaves a
15955    /// record for `land`; one with only a Minor, or no reject, leaves none.
15956    #[tokio::test]
15957    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15958        use crate::verdict::{Finding, ReviewVote, Severity};
15959        crate::run::pin_test_home();
15960        let tmp = tempfile::tempdir().expect("tempdir");
15961        let repo = tmp.path().join("repo");
15962        std::fs::create_dir_all(&repo).unwrap();
15963        init_repo(&repo);
15964
15965        for (severity, vote, expect) in [
15966            (Severity::Major, ReviewVote::Reject, true),
15967            (Severity::Minor, ReviewVote::Reject, false),
15968            (Severity::Major, ReviewVote::Approve, false),
15969        ] {
15970            let mut round = review_round(false, 1, 1, 1, false, true);
15971            round.reviews = vec![ReviewRecord {
15972                reviewer: 1,
15973                agent: "alpha".to_owned(),
15974                summary: String::new(),
15975                findings: vec![Finding {
15976                    id: "R1-1-1".to_owned(),
15977                    severity,
15978                    file: None,
15979                    line: None,
15980                    title: "t".to_owned(),
15981                    detail: String::new(),
15982                }],
15983                vote: Some(vote),
15984                failed: None,
15985                duration_ms: 0,
15986                attempts: 0,
15987            }];
15988            let mut state = RunState::new(
15989                repo.clone(),
15990                "main".to_owned(),
15991                "deadbeef".to_owned(),
15992                "task".to_owned(),
15993                Config::default(),
15994            );
15995            state.reviews = vec![round];
15996            let mut runner = Runner {
15997                state,
15998                roles: ResolvedRoles {
15999                    implementers: Vec::new(),
16000                    judges: Vec::new(),
16001                    reviewers: Vec::new(),
16002                    fixer: None,
16003                    conductor: conductor(),
16004                    implementer_roster: Vec::new(),
16005                    judge_roster: Vec::new(),
16006                    reviewer_roster: Vec::new(),
16007                },
16008                sem: Arc::new(Semaphore::new(1)),
16009                pause: Pause::new(),
16010                interrupt: Pause::new(),
16011            };
16012            let shell = runner.state.config.shell();
16013            runner
16014                .stop_reviewing("round budget spent", &shell, &repo)
16015                .await
16016                .expect("stop_reviewing");
16017            assert_eq!(runner.state.status, RunStatus::Gating);
16018            assert_eq!(
16019                runner.state.contested_handoff.is_some(),
16020                expect,
16021                "{severity:?} + {vote:?}"
16022            );
16023        }
16024    }
16025
16026    /// The shape the incident this whole fix responds to actually had: the
16027    /// round budget spent, the last round's own e2e blocked on the shared
16028    /// build cache (held here by a live pid — this test process — exactly
16029    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16030    /// without forking a process). `stop_reviewing` must retry it — not
16031    /// silently leave the round looking untouched (the catch-up-only half of
16032    /// the bug), and not read the contention as a red `e2e` and block the
16033    /// run on it (the other half). Called directly, the same way
16034    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16035    /// above exercises `gate`, so this never needs a real cargo build to
16036    /// reach: the lease is never released, so `with_cache_lease` never gets
16037    /// past acquiring it into anything that would need a real workspace.
16038    #[tokio::test]
16039    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16040        crate::run::pin_test_home();
16041        let home = crate::run::home();
16042
16043        let tmp = tempfile::tempdir().expect("tempdir");
16044        let repo = tmp.path().join("repo");
16045        std::fs::create_dir_all(&repo).unwrap();
16046        init_repo(&repo);
16047        let head = crate::git::rev_parse(&repo, "HEAD")
16048            .await
16049            .expect("rev-parse");
16050        // Unique to this test, so holding its lease cannot collide with
16051        // another test sharing the same process-wide `home`.
16052        let cache_dir = tmp.path().join("target");
16053
16054        let mut config = Config::default();
16055        config.verify.e2e = vec![format!(
16056            "CARGO_TARGET_DIR='{}' test -f README.md",
16057            cache_dir.display()
16058        )];
16059        config.graph.review_rounds = 1;
16060        // Bounded so a regression that does start waiting fails the test in
16061        // seconds, not hangs it.
16062        config.graph.timeout_verify = Some(2);
16063        config.graph.timeout_resource_wait = 2;
16064
16065        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16066        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16067            .expect("no io error acquiring directly")
16068        {
16069            crate::cache::AcquireOutcome::Acquired(g) => g,
16070            crate::cache::AcquireOutcome::Busy(b) => {
16071                panic!("expected the direct acquire to win the lease first: {b:?}")
16072            }
16073        };
16074
16075        let mut state = RunState::new(
16076            repo.clone(),
16077            "main".to_owned(),
16078            head.clone(),
16079            "task".to_owned(),
16080            config,
16081        );
16082        state.candidates = vec![Candidate {
16083            index: 0,
16084            label: 'A',
16085            agent: "alpha".to_owned(),
16086            branch: "does-not-exist".to_owned(),
16087            worktree: repo.clone(),
16088            summary: String::new(),
16089            stat: String::new(),
16090            files: 0,
16091            commits: 0,
16092            empty: false,
16093            failed: None,
16094            verified_noop: None,
16095            duration_ms: 0,
16096            folded: false,
16097        }];
16098        state.tally = Some(Tally {
16099            first_choice: BTreeMap::from([('A', 1)]),
16100            borda: BTreeMap::new(),
16101            winner: 'A',
16102            rankings: 1,
16103            unanimous_initial: true,
16104            deliberated: false,
16105            changed_votes: 0,
16106            unanimous_final: true,
16107            tie_break: None,
16108            judges: 0,
16109            present: 0,
16110            quorum: 0,
16111            met_quorum: true,
16112            uncontested: Some("only candidate A produced a change".to_owned()),
16113        });
16114        // The round budget's last round, deferred: `needs_catchup_run`'s
16115        // other trigger. `stop_reviewing`'s retry machinery must treat this
16116        // exactly like a resource-blocked attempt once it actually runs.
16117        state.reviews = vec![ReviewRound {
16118            round: 1,
16119            head: head.clone(),
16120            verified_head: None,
16121            verified_at: None,
16122            reviews: Vec::new(),
16123            e2e: Vec::new(),
16124            fix: None,
16125            blocking: 1,
16126            answered: 1,
16127            expected: 1,
16128            clean: false,
16129            verify_retried: false,
16130            e2e_deferred: true,
16131            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16132            progressed: false,
16133            vote_split: false,
16134            reconsideration: Vec::new(),
16135            verdict: None,
16136        }];
16137
16138        let mut runner = Runner {
16139            state,
16140            roles: ResolvedRoles {
16141                implementers: Vec::new(),
16142                judges: Vec::new(),
16143                reviewers: Vec::new(),
16144                fixer: None,
16145                conductor: conductor(),
16146                implementer_roster: Vec::new(),
16147                judge_roster: Vec::new(),
16148                reviewer_roster: Vec::new(),
16149            },
16150            sem: Arc::new(Semaphore::new(1)),
16151            pause: Pause::new(),
16152            interrupt: Pause::new(),
16153        };
16154
16155        let shell = runner.state.config.shell();
16156        runner
16157            .stop_reviewing("round budget spent", &shell, &repo)
16158            .await
16159            .expect("stop_reviewing");
16160
16161        let last = runner.state.reviews.last().expect("round record");
16162        assert_eq!(
16163            last.e2e_status(),
16164            E2eStatus::ResourceBlocked,
16165            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16166             failed: {last:?}"
16167        );
16168        assert_eq!(
16169            last.verified_head.as_deref(),
16170            Some(head.as_str()),
16171            "which commit this attempt targeted is known even though nothing finished checking \
16172             it"
16173        );
16174        let first_attempt_at = last
16175            .verified_at
16176            .expect("when this attempt ran is known too");
16177        assert_ne!(
16178            runner.state.status,
16179            RunStatus::Blocked,
16180            "contention is evidence about the machine, not the patch — it must not settle the \
16181             run as blocked: {:?}",
16182            runner.state.status
16183        );
16184        assert!(
16185            !runner
16186                .state
16187                .events
16188                .iter()
16189                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16190            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16191            runner.state.events
16192        );
16193
16194        // The cache is still held: a later reentry must retry the same
16195        // round's verification again — not leave it looking exactly as
16196        // untouched as the first blocked attempt, which is indistinguishable
16197        // from never having tried again at all.
16198        runner
16199            .stop_reviewing("round budget spent", &shell, &repo)
16200            .await
16201            .expect("stop_reviewing retry");
16202        assert_eq!(
16203            runner.state.reviews.len(),
16204            1,
16205            "no new round was started: {:?}",
16206            runner.state.reviews
16207        );
16208        let last = runner.state.reviews.last().expect("round record");
16209        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16210        assert!(
16211            last.verified_at.expect("still known") > first_attempt_at,
16212            "a second reentry must be a fresh attempt, not a stale copy of the first"
16213        );
16214        assert_ne!(runner.state.status, RunStatus::Blocked);
16215
16216        held.release();
16217    }
16218
16219    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
16220    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
16221    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
16222    /// own top-of-function fast path (`review_conclusion`) correctly reads
16223    /// this shape as `None` rather than guessing `Blocked`, and the loop's
16224    /// own `for` range is empty once the round budget is spent, so
16225    /// `review_loop` must retry the check itself rather than silently doing
16226    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
16227    /// above exercises directly, but through `review_loop`'s own entry point
16228    /// this time, proving the wiring between the two rather than just the
16229    /// retry logic in isolation.
16230    #[tokio::test]
16231    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16232        crate::run::pin_test_home();
16233        let home = crate::run::home();
16234
16235        let tmp = tempfile::tempdir().expect("tempdir");
16236        let repo = tmp.path().join("repo");
16237        std::fs::create_dir_all(&repo).unwrap();
16238        init_repo(&repo);
16239        let head = crate::git::rev_parse(&repo, "HEAD")
16240            .await
16241            .expect("rev-parse");
16242        let cache_dir = tmp.path().join("target");
16243
16244        let mut config = Config::default();
16245        config.verify.e2e = vec![format!(
16246            "CARGO_TARGET_DIR='{}' test -f README.md",
16247            cache_dir.display()
16248        )];
16249        config.graph.review_rounds = 1;
16250        config.graph.timeout_verify = Some(2);
16251        config.graph.timeout_resource_wait = 2;
16252
16253        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16254        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16255            .expect("no io error acquiring directly")
16256        {
16257            crate::cache::AcquireOutcome::Acquired(g) => g,
16258            crate::cache::AcquireOutcome::Busy(b) => {
16259                panic!("expected the direct acquire to win the lease first: {b:?}")
16260            }
16261        };
16262
16263        let mut state = RunState::new(
16264            repo.clone(),
16265            "main".to_owned(),
16266            head.clone(),
16267            "task".to_owned(),
16268            config,
16269        );
16270        state.candidates = vec![Candidate {
16271            index: 0,
16272            label: 'A',
16273            agent: "alpha".to_owned(),
16274            branch: "does-not-exist".to_owned(),
16275            worktree: repo.clone(),
16276            summary: String::new(),
16277            stat: String::new(),
16278            files: 0,
16279            commits: 0,
16280            empty: false,
16281            failed: None,
16282            verified_noop: None,
16283            duration_ms: 0,
16284            folded: false,
16285        }];
16286        state.tally = Some(Tally {
16287            first_choice: BTreeMap::from([('A', 1)]),
16288            borda: BTreeMap::new(),
16289            winner: 'A',
16290            rankings: 1,
16291            unanimous_initial: true,
16292            deliberated: false,
16293            changed_votes: 0,
16294            unanimous_final: true,
16295            tie_break: None,
16296            judges: 0,
16297            present: 0,
16298            quorum: 0,
16299            met_quorum: true,
16300            uncontested: Some("only candidate A produced a change".to_owned()),
16301        });
16302        // The exact shape a prior process's `stop_reviewing` would have left
16303        // on disk: the round budget's last round, a real attempt already
16304        // made and already resource-blocked.
16305        state.reviews = vec![ReviewRound {
16306            round: 1,
16307            head: head.clone(),
16308            verified_head: Some(head.clone()),
16309            verified_at: Some(jiff::Timestamp::now()),
16310            reviews: Vec::new(),
16311            e2e: vec![CommandOutcome {
16312                command: format!(
16313                    "CARGO_TARGET_DIR='{}' test -f README.md",
16314                    cache_dir.display()
16315                ),
16316                code: None,
16317                output_tail: "waiting for the shared build cache".to_owned(),
16318                duration_ms: 0,
16319                resource_blocked: true,
16320            }],
16321            fix: None,
16322            blocking: 1,
16323            answered: 1,
16324            expected: 1,
16325            clean: false,
16326            verify_retried: false,
16327            e2e_deferred: false,
16328            e2e_defer_reason: None,
16329            progressed: false,
16330            vote_split: false,
16331            reconsideration: Vec::new(),
16332            verdict: None,
16333        }];
16334
16335        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16336        let mut runner = Runner {
16337            state,
16338            roles: ResolvedRoles {
16339                implementers: Vec::new(),
16340                judges: Vec::new(),
16341                reviewers: Vec::new(),
16342                fixer: None,
16343                conductor: conductor(),
16344                implementer_roster: Vec::new(),
16345                judge_roster: Vec::new(),
16346                reviewer_roster: Vec::new(),
16347            },
16348            sem: Arc::new(Semaphore::new(1)),
16349            pause: Pause::new(),
16350            interrupt: Pause::new(),
16351        };
16352
16353        // The lease is still held throughout, so this reentry's own retry is
16354        // also contended — proving `review_loop` actually tried again (not
16355        // that it happened to succeed) is what the timestamp comparison
16356        // below is for.
16357        runner.review_loop().await.expect("review_loop");
16358
16359        assert_eq!(
16360            runner.state.reviews.len(),
16361            1,
16362            "no new round was started on top of the unresolved one: {:?}",
16363            runner.state.reviews
16364        );
16365        let last = &runner.state.reviews[0];
16366        assert_eq!(
16367            last.e2e_status(),
16368            E2eStatus::ResourceBlocked,
16369            "still contended: {last:?}"
16370        );
16371        assert!(
16372            last.verified_at.expect("still known") > first_attempt_at,
16373            "review_loop must have actually retried the check, not left it exactly as found"
16374        );
16375        assert_ne!(
16376            runner.state.status,
16377            RunStatus::Blocked,
16378            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16379            runner.state.status
16380        );
16381
16382        held.release();
16383    }
16384
16385    #[tokio::test]
16386    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16387        crate::run::pin_test_home();
16388        let tmp = tempfile::tempdir().expect("tempdir");
16389        let repo = tmp.path().join("repo");
16390        std::fs::create_dir_all(&repo).unwrap();
16391        init_repo(&repo);
16392
16393        let mut config = Config::default();
16394        config.merge.mode = MergeMode::Pr;
16395        config.graph.land = true;
16396        config.graph.land_approval = false;
16397
16398        let mut state = RunState::new(
16399            repo.clone(),
16400            "main".to_owned(),
16401            "deadbeef".to_owned(),
16402            "task".to_owned(),
16403            config,
16404        );
16405        state.candidates = vec![Candidate {
16406            index: 0,
16407            label: 'A',
16408            agent: "alpha".to_owned(),
16409            branch: "does-not-exist".to_owned(),
16410            worktree: repo.clone(),
16411            summary: String::new(),
16412            stat: String::new(),
16413            files: 0,
16414            commits: 0,
16415            empty: false,
16416            failed: None,
16417            verified_noop: None,
16418            duration_ms: 0,
16419            folded: false,
16420        }];
16421        state.tally = Some(Tally {
16422            first_choice: BTreeMap::from([('A', 1)]),
16423            borda: BTreeMap::new(),
16424            winner: 'A',
16425            rankings: 1,
16426            unanimous_initial: true,
16427            deliberated: false,
16428            changed_votes: 0,
16429            unanimous_final: true,
16430            tie_break: None,
16431            judges: 0,
16432            present: 0,
16433            quorum: 0,
16434            met_quorum: true,
16435            uncontested: Some("only candidate A produced a change".to_owned()),
16436        });
16437        state.reviews = vec![ReviewRound {
16438            round: 1,
16439            head: "deadbeef".to_owned(),
16440            verified_head: None,
16441            verified_at: None,
16442            reviews: Vec::new(),
16443            e2e: Vec::new(),
16444            fix: None,
16445            blocking: 0,
16446            answered: 0,
16447            expected: 0,
16448            clean: true,
16449            verify_retried: false,
16450            e2e_deferred: false,
16451            e2e_defer_reason: None,
16452            progressed: false,
16453            vote_split: false,
16454            reconsideration: Vec::new(),
16455            verdict: None,
16456        }];
16457        state.gate = vec![CommandOutcome {
16458            command: "test".to_owned(),
16459            code: Some(0),
16460            output_tail: String::new(),
16461            duration_ms: 0,
16462            resource_blocked: false,
16463        }];
16464        state.gate_ran = true;
16465        // A first pass through `merge` already pushed and opened this pull
16466        // request; `status` is `Landing` because a previous call into `land`
16467        // parked or was interrupted before it reached a terminal outcome.
16468        state.status = RunStatus::Landing;
16469        state.merge = Some(MergeOutcome {
16470            mode: MergeMode::Pr,
16471            ok: true,
16472            detail: "https://example.invalid/x/y/pull/1".to_owned(),
16473            empty: false,
16474        });
16475
16476        // The Landing-resume shortcut calls `run_land` directly rather than
16477        // through `merge`, which is exactly the call site that used to skip
16478        // `settle_questions` - see the fixture below.
16479        ask_test_home();
16480        // A real resume loads a persisted run; driver ownership is checked
16481        // against that record before the Landing shortcut is entered.
16482        state.save().expect("save landing run");
16483        let store = ask::Questions::open();
16484        let q = ask_open_question(&store, &state.id);
16485
16486        let mut runner = Runner {
16487            state,
16488            roles: ResolvedRoles {
16489                implementers: Vec::new(),
16490                judges: Vec::new(),
16491                reviewers: Vec::new(),
16492                fixer: None,
16493                conductor: conductor(),
16494                implementer_roster: Vec::new(),
16495                judge_roster: Vec::new(),
16496                reviewer_roster: Vec::new(),
16497            },
16498            sem: Arc::new(Semaphore::new(1)),
16499            pause: Pause::new(),
16500            interrupt: Pause::new(),
16501        };
16502
16503        // `execute`, not `merge` directly: the Landing-resume shortcut lives
16504        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
16505        // exactly because `review_loop` would otherwise clobber the marker
16506        // first.
16507        runner.execute().await.expect("execute");
16508
16509        assert_eq!(
16510            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16511            Some("https://example.invalid/x/y/pull/1"),
16512            "reentry must not push again or open a second pull request over the \
16513             one `land` is already watching"
16514        );
16515        assert_ne!(
16516            runner.state.status,
16517            RunStatus::Landing,
16518            "land could not actually reach the fake pull request, so it must \
16519             have given up rather than left the run silently parked forever"
16520        );
16521        // `land` could not reach the fake pull request, so it gave up into
16522        // `Blocked` - still resumable, so the question must not have been
16523        // swept just because this branch now also calls `settle_questions`.
16524        assert_eq!(runner.state.status, RunStatus::Blocked);
16525        assert!(
16526            store.get(&q.id).unwrap().status.open(),
16527            "Blocked is still alive; settle_questions must have been a no-op here"
16528        );
16529    }
16530
16531    fn state_with_round(round: ReviewRound) -> RunState {
16532        let mut s = RunState::new(
16533            PathBuf::from("/repo"),
16534            "main".to_owned(),
16535            "abc1234".to_owned(),
16536            "add retries".to_owned(),
16537            Config::default(),
16538        );
16539        s.reviews = vec![round];
16540        s
16541    }
16542
16543    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16544        crate::verdict::Finding {
16545            id: id.to_owned(),
16546            severity,
16547            file: None,
16548            line: None,
16549            title: title.to_owned(),
16550            detail: String::new(),
16551        }
16552    }
16553
16554    #[test]
16555    fn pr_body_names_open_findings_and_declined_ones() {
16556        let round = ReviewRound {
16557            round: 2,
16558            head: "deadbee".to_owned(),
16559            verified_head: None,
16560            verified_at: None,
16561            reviews: vec![ReviewRecord {
16562                attempts: 0,
16563                reviewer: 1,
16564                agent: "alpha".to_owned(),
16565                summary: String::new(),
16566                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16567                vote: None,
16568                failed: None,
16569                duration_ms: 0,
16570            }],
16571            e2e: vec![CommandOutcome {
16572                command: "cargo test".to_owned(),
16573                code: Some(0),
16574                output_tail: String::new(),
16575                duration_ms: 0,
16576                resource_blocked: false,
16577            }],
16578            verify_retried: false,
16579            e2e_deferred: false,
16580            e2e_defer_reason: None,
16581            fix: Some(FixRecord {
16582                agent: "alpha".to_owned(),
16583                addressed: Vec::new(),
16584                rejected: vec![crate::verdict::Rejection {
16585                    id: "R1-1-1".to_owned(),
16586                    why: "not reachable from any caller".to_owned(),
16587                }],
16588                notes: String::new(),
16589                committed: true,
16590                failed: None,
16591                duration_ms: 0,
16592                continuation: None,
16593            }),
16594            blocking: 0,
16595            answered: 1,
16596            expected: 1,
16597            clean: false,
16598            progressed: true,
16599            vote_split: false,
16600            reconsideration: Vec::new(),
16601            verdict: None,
16602        };
16603        let state = state_with_round(round);
16604        let body = pr_message(&state, 'A').body;
16605
16606        assert!(body.contains("add retries"), "the task must still be there");
16607        assert!(body.contains("R2-1-1"), "{body}");
16608        assert!(!body.contains("unused import"), "{body}");
16609        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16610        assert!(
16611            !body.contains("not reachable from any caller"),
16612            "the reason it was declined: {body}"
16613        );
16614    }
16615
16616    #[test]
16617    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16618        let round = ReviewRound {
16619            round: 1,
16620            head: "deadbee".to_owned(),
16621            verified_head: None,
16622            verified_at: None,
16623            reviews: vec![ReviewRecord {
16624                attempts: 0,
16625                reviewer: 1,
16626                agent: "alpha".to_owned(),
16627                summary: String::new(),
16628                findings: Vec::new(),
16629                vote: None,
16630                failed: None,
16631                duration_ms: 0,
16632            }],
16633            e2e: Vec::new(),
16634            verify_retried: false,
16635            e2e_deferred: false,
16636            e2e_defer_reason: None,
16637            fix: None,
16638            blocking: 0,
16639            answered: 1,
16640            expected: 1,
16641            clean: true,
16642            progressed: false,
16643            vote_split: false,
16644            reconsideration: Vec::new(),
16645            verdict: None,
16646        };
16647        let state = state_with_round(round);
16648        let body = pr_message(&state, 'A').body;
16649        assert!(!body.contains("Open review findings"), "{body}");
16650        assert!(!body.contains("Declined"), "{body}");
16651    }
16652
16653    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16654        let mut state = RunState::new(
16655            PathBuf::from("/repo"),
16656            "main".to_owned(),
16657            "abc1234".to_owned(),
16658            instruction.to_owned(),
16659            Config::default(),
16660        );
16661        state.candidates.push(Candidate {
16662            index: 0,
16663            label: 'A',
16664            agent: "alpha".to_owned(),
16665            branch: "magi/x/A".to_owned(),
16666            worktree: PathBuf::from("/wt"),
16667            summary: summary.to_owned(),
16668            stat: String::new(),
16669            files: 1,
16670            commits: 1,
16671            empty: false,
16672            failed: None,
16673            verified_noop: None,
16674            folded: false,
16675            duration_ms: 0,
16676        });
16677        state
16678    }
16679
16680    fn review_state(subjects: &[&str]) -> RunState {
16681        let mut state = state_with_summary(
16682            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16683            "",
16684        );
16685        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16686        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16687        state
16688    }
16689
16690    /// A branch rebased onto a main that moved past the recorded
16691    /// `base_commit` is titled from its own first commit, never main's.
16692    #[tokio::test]
16693    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16694        ask_test_home();
16695        let tmp = tempfile::tempdir().unwrap();
16696        let repo = tmp.path().join("repo");
16697        std::fs::create_dir_all(&repo).unwrap();
16698        init_repo(&repo);
16699        let origin = tmp.path().join("origin.git");
16700        let g = |dir: &Path, args: &[&str]| {
16701            let out = std::process::Command::new("git")
16702                .args(args)
16703                .current_dir(dir)
16704                .quiet()
16705                .output()
16706                .expect("spawn git");
16707            assert!(
16708                out.status.success(),
16709                "git {args:?}: {}",
16710                String::from_utf8_lossy(&out.stderr)
16711            );
16712        };
16713        g(
16714            tmp.path(),
16715            &[
16716                "clone",
16717                "--bare",
16718                "-q",
16719                repo.to_str().unwrap(),
16720                origin.to_str().unwrap(),
16721            ],
16722        );
16723        g(
16724            &repo,
16725            &["remote", "add", "origin", origin.to_str().unwrap()],
16726        );
16727        let c1 = git::rev_parse(&repo, "main").await.unwrap();
16728
16729        // Main moves on; the branch is built on top of the new main.
16730        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16731        g(&repo, &["add", "-A"]);
16732        g(
16733            &repo,
16734            &["commit", "-q", "-m", "chore(deps): update a crate"],
16735        );
16736        g(&repo, &["push", "-q", "origin", "main"]);
16737        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16738        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16739        g(&repo, &["add", "-A"]);
16740        g(
16741            &repo,
16742            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16743        );
16744        g(&repo, &["checkout", "-q", "main"]);
16745
16746        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16747        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16748        // Without a readable tracking ref the recorded base's merge base is used.
16749        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16750        assert_eq!(fallback, c1);
16751
16752        let mut state = review_state(&[
16753            "chore(deps): update a crate",
16754            "fix(daemon): apply a chosen action",
16755        ]);
16756        state.repo = repo.clone();
16757        state.base_branch = "main".to_owned();
16758        state.base_commit = c1;
16759        refresh_reviewed_commits(&mut state, "feat/own").await;
16760        assert_eq!(
16761            state.reviewed_commits,
16762            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16763        );
16764        assert_eq!(
16765            review_title(&state).as_deref(),
16766            Some("fix(daemon): apply a chosen action")
16767        );
16768        assert_eq!(
16769            leaked_subjects(&state, "feat/own").await,
16770            Some(vec!["chore(deps): update a crate".to_owned()])
16771        );
16772        // A stale tracking ref is still used when the fetch fails, but the
16773        // leak list is withheld.
16774        state.config.merge.remote = "nowhere".to_owned();
16775        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16776    }
16777
16778    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
16779    /// with `origin/main` one commit behind it, and a summarizing agent that
16780    /// prints whatever `reply` holds.
16781    struct SummaryFixture {
16782        _tmp: tempfile::TempDir,
16783        repo: PathBuf,
16784        reply: PathBuf,
16785        runner: Runner,
16786    }
16787
16788    fn summary_fixture() -> SummaryFixture {
16789        ask_test_home();
16790        let tmp = tempfile::tempdir().unwrap();
16791        let repo = tmp.path().join("repo");
16792        std::fs::create_dir_all(&repo).unwrap();
16793        init_repo(&repo);
16794        let origin = tmp.path().join("origin.git");
16795        let g = |args: &[&str]| {
16796            let out = std::process::Command::new("git")
16797                .args(args)
16798                .current_dir(&repo)
16799                .quiet()
16800                .output()
16801                .expect("spawn git");
16802            assert!(
16803                out.status.success(),
16804                "git {args:?}: {}",
16805                String::from_utf8_lossy(&out.stderr)
16806            );
16807        };
16808        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16809        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16810        g(&["push", "-q", "origin", "main"]);
16811        g(&["checkout", "-q", "-b", "feat/own"]);
16812        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16813        g(&["add", "-A"]);
16814        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16815        let reply = tmp.path().join("reply.txt");
16816        std::fs::write(
16817            &reply,
16818            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16819        )
16820        .unwrap();
16821        let mut runner = runner_at(RunStatus::Gating);
16822        let mut state = review_state(&["feat(own): add the own file"]);
16823        state.repo = repo.clone();
16824        state.base_branch = "main".to_owned();
16825        state.candidates[0].branch = "feat/own".to_owned();
16826        state.candidates[0].worktree = repo.clone();
16827        let mut author = spec("alpha");
16828        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16829        author
16830            .env
16831            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16832        state.config.agents = vec![author];
16833        runner.state = state;
16834        SummaryFixture {
16835            _tmp: tmp,
16836            repo,
16837            reply,
16838            runner,
16839        }
16840    }
16841
16842    async fn posted(f: &mut SummaryFixture) -> PrMessage {
16843        let winner = f.runner.state.candidates[0].clone();
16844        f.runner
16845            .guarded_pr_message(&winner, None, true)
16846            .await
16847            .unwrap()
16848            .expect("an English description never parks")
16849    }
16850
16851    fn clean_round_with_minor_findings() -> ReviewRound {
16852        let mut round = passing_round(2, None);
16853        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16854        round.reviews = vec![ReviewRecord {
16855            attempts: 0,
16856            reviewer: 1,
16857            agent: "alpha".into(),
16858            summary: "内部レビュー".into(),
16859            findings: vec![
16860                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16861                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16862            ],
16863            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16864            failed: None,
16865            duration_ms: 0,
16866        }];
16867        round
16868    }
16869
16870    #[tokio::test]
16871    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16872        for review_only in [false, true] {
16873            let mut f = summary_fixture();
16874            if !review_only {
16875                f.runner.state.candidates[0].agent = "alpha".into();
16876            }
16877            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16878            assert!(f.runner.state.reviews[0].clean);
16879            assert!(f.runner.state.reviews[0].fix.is_none());
16880            assert!(f.runner.state.open_findings().is_empty());
16881            assert_eq!(
16882                crate::pr_outcome::required_finding_ids(&f.runner.state),
16883                ["R2-1-1", "R2-1-2"]
16884            );
16885            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16886            let evidence = crate::pr_outcome::summary_evidence(
16887                &f.runner.state,
16888                crate::pr_outcome::FinalHead::At(&head),
16889            );
16890            assert!(
16891                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16892            );
16893            // The ordinary reply omits both IDs: no cached description is adopted.
16894            let omitted = posted(&mut f).await;
16895            assert!(omitted.body.contains("No description"));
16896            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16897            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16898            assert!(f.runner.state.change_summary.is_none());
16899            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16900            let described = posted(&mut f).await;
16901            assert!(described.body.contains("improve documentation for callers"));
16902            assert!(described.body.contains("remaining edge case"));
16903            assert!(described.body.contains("## Open review findings"));
16904            assert!(!described.body.contains("未修正"));
16905        }
16906    }
16907
16908    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16909        let mut f = summary_fixture();
16910        if !review_only {
16911            f.runner.state.candidates[0].agent = "alpha".into();
16912        }
16913        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16914        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16915        fixed
16916            .fix
16917            .as_mut()
16918            .unwrap()
16919            .rejected
16920            .push(crate::verdict::Rejection {
16921                id: "R1-1-1".into(),
16922                why: "呼び出されないコードのため辞退".into(),
16923            });
16924        let mut clean = clean_round_with_minor_findings();
16925        clean.verified_head = Some(head.clone());
16926        f.runner.state.reviews = vec![fixed, clean];
16927        ok_gate(&mut f.runner.state, Some(&head));
16928        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16929        f.runner.state.config.agents.push(scripted("beta", reply));
16930        f.runner.state.config.roles.language_judge =
16931            Some(crate::config::AgentChoice::One("beta".into()));
16932        f
16933    }
16934
16935    #[tokio::test]
16936    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16937        for review_only in [false, true] {
16938            let mut f = rewrite_fixture_with_findings(review_only,
16939                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16940            let m = posted(&mut f).await;
16941            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16942            let expected =
16943                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16944            assert!(m.body.contains(&expected));
16945            assert!(m.body.contains("## Review rounds"));
16946            assert!(
16947                m.body
16948                    .contains("Gate (`verify.gate`): passed on the final commit")
16949            );
16950            assert!(
16951                m.body
16952                    .contains("Tests (`verify.e2e`): passed on the final commit")
16953            );
16954            let c = f.runner.state.change_summary.as_ref().unwrap();
16955            assert!(
16956                m.body
16957                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16958            );
16959            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16960            assert!(crate::pr_outcome::covers_required_findings(
16961                &f.runner.state,
16962                &m.body
16963            ));
16964            assert!(m.body.contains("documentation remains incomplete"));
16965            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16966            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16967            assert!(f.runner.state.github_text.is_none());
16968        }
16969    }
16970
16971    #[tokio::test]
16972    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16973        for review_only in [false, true] {
16974            for reply in [
16975                "TITLE: fix: public description\nA short English description.",
16976                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16977            ] {
16978                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16979                let winner = f.runner.state.candidates[0].clone();
16980                let posted = f
16981                    .runner
16982                    .guarded_pr_message(&winner, None, true)
16983                    .await
16984                    .unwrap();
16985                assert!(posted.is_none());
16986                assert!(f.runner.state.parked);
16987                let withheld = f.runner.state.github_text.as_ref().unwrap();
16988                assert!(
16989                    withheld.rewrite.is_none(),
16990                    "an incomplete rewrite must not be selectable"
16991                );
16992                let q = ask::Questions::open()
16993                    .get(withheld.question.as_ref().unwrap())
16994                    .unwrap();
16995                assert!(
16996                    !q.choices
16997                        .contains(&crate::github_text::USE_REWRITE.to_owned())
16998                );
16999                assert!(
17000                    q.detail.contains("R1-1-1")
17001                        && q.detail.contains("R2-1-1")
17002                        && q.detail.contains("R2-1-2")
17003                );
17004            }
17005        }
17006    }
17007
17008    #[tokio::test]
17009    async fn japanese_reports_stay_local_in_both_public_run_modes() {
17010        for review_only in [false, true] {
17011            let mut f = summary_fixture();
17012            if !review_only {
17013                f.runner.state.candidates[0].agent = "alpha".into();
17014            }
17015            f.runner.state.instruction = "内部タスクの説明".into();
17016            f.runner.state.candidates[0].summary = "内部の候補報告".into();
17017            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17018            f.runner.state.reviews = vec![
17019                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17020                round_with_notes(2, Some("二回目の内部修正報告です。")),
17021                passing_round(3, Some(&head)),
17022            ];
17023            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17024            ok_gate(&mut f.runner.state, Some(&head));
17025            let captured = f.repo.join("public-prompt.txt");
17026            let author = &mut f.runner.state.config.agents[0];
17027            author.command = vec![
17028                "sh".into(),
17029                "-c".into(),
17030                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17031            ];
17032            author.env.insert(
17033                "PROMPT_FILE".into(),
17034                captured.to_string_lossy().into_owned(),
17035            );
17036            let m = posted(&mut f).await;
17037            assert!(m.body.contains("Adds `own.txt`"));
17038            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17039            assert!(m.body.contains("R1-1-1"));
17040            assert!(
17041                m.body
17042                    .contains("Gate (`verify.gate`): passed on the final commit")
17043            );
17044            assert!(
17045                m.body
17046                    .contains("Tests (`verify.e2e`): passed on the final commit")
17047            );
17048            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17049            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17050            let prompt = std::fs::read_to_string(captured).unwrap();
17051            assert!(prompt.contains(&format!(
17052                "git diff {} {head}",
17053                f.runner.state.change_summary.as_ref().unwrap().base
17054            )));
17055            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17056            assert_eq!(
17057                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17058                "二回目の内部修正報告です。"
17059            );
17060        }
17061    }
17062
17063    #[tokio::test]
17064    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17065        for review_only in [false, true] {
17066            let mut f = summary_fixture();
17067            if !review_only {
17068                f.runner.state.candidates[0].agent = "alpha".into();
17069            }
17070            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17071            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17072            std::fs::write(&f.reply, "no usable description").unwrap();
17073            let failed = posted(&mut f).await;
17074            assert!(
17075                failed
17076                    .body
17077                    .contains("No description of what this change does could be generated")
17078            );
17079            assert!(failed.body.is_ascii());
17080            assert!(f.runner.state.github_text.is_none());
17081            assert!(f.runner.state.change_summary.is_none());
17082            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17083            let evidence = crate::pr_outcome::summary_evidence(
17084                &f.runner.state,
17085                crate::pr_outcome::FinalHead::At(&head),
17086            );
17087            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17088            assert!(
17089                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17090                    .exists()
17091            );
17092            f.runner.state =
17093                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17094            std::fs::write(
17095                &f.reply,
17096                "TITLE: feat(own): add the own file\nAdds the committed own file.",
17097            )
17098            .unwrap();
17099            let retried = posted(&mut f).await;
17100            assert!(retried.body.contains("Adds the committed own file."));
17101            assert_eq!(
17102                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17103                "日本語の修正報告"
17104            );
17105        }
17106    }
17107
17108    #[tokio::test]
17109    async fn same_head_verification_and_findings_invalidate_public_prose() {
17110        let mut f = summary_fixture();
17111        posted(&mut f).await;
17112        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17113        let original = f
17114            .runner
17115            .state
17116            .change_summary
17117            .as_ref()
17118            .unwrap()
17119            .evidence
17120            .clone();
17121        ok_gate(&mut f.runner.state, Some(&head));
17122        std::fs::write(
17123            &f.reply,
17124            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17125        )
17126        .unwrap();
17127        let verified = posted(&mut f).await;
17128        assert!(verified.body.contains("now verified"));
17129        assert_ne!(
17130            f.runner.state.change_summary.as_ref().unwrap().evidence,
17131            original
17132        );
17133        let mut round = round_with_notes(1, Some("内部報告"));
17134        round.reviews.push(ReviewRecord {
17135            attempts: 0,
17136            reviewer: 1,
17137            agent: "alpha".into(),
17138            summary: "内部レビュー".into(),
17139            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17140            vote: None,
17141            failed: None,
17142            duration_ms: 0,
17143        });
17144        round
17145            .fix
17146            .as_mut()
17147            .unwrap()
17148            .rejected
17149            .push(crate::verdict::Rejection {
17150                id: "R1-1-1".into(),
17151                why: "呼び出し元から到達できない".into(),
17152            });
17153        f.runner.state.reviews = vec![round];
17154        // A description omitting the required ID is unusable, even with a title.
17155        let omitted = posted(&mut f).await;
17156        assert!(omitted.body.contains("No description"));
17157        assert!(omitted.body.contains("R1-1-1"));
17158        assert!(!omitted.body.contains("到達"));
17159        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17160        let described = posted(&mut f).await;
17161        assert!(described.body.contains("unreachable from callers"));
17162        assert!(described.body.contains("R1-1-1"));
17163        assert!(!described.body.contains("到達"));
17164        let evidence = f
17165            .runner
17166            .state
17167            .change_summary
17168            .as_ref()
17169            .unwrap()
17170            .evidence
17171            .clone();
17172        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17173        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17174        assert!(posted(&mut f).await.body.contains("rationale updated"));
17175        assert_ne!(
17176            f.runner.state.change_summary.as_ref().unwrap().evidence,
17177            evidence
17178        );
17179        // Old serialized summaries have no evidence fingerprint and must regenerate.
17180        f.runner
17181            .state
17182            .change_summary
17183            .as_mut()
17184            .unwrap()
17185            .evidence
17186            .clear();
17187        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17188        assert!(posted(&mut f).await.body.contains("current specification"));
17189    }
17190
17191    #[tokio::test]
17192    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17193        let mut f = summary_fixture();
17194        let m = posted(&mut f).await;
17195        assert_eq!(m.title, "feat(own): add the own file");
17196        assert!(
17197            m.body
17198                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17199            "{}",
17200            m.body
17201        );
17202        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17203        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17204        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17205        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17206        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17207        let kept = f.runner.state.change_summary.as_ref().unwrap();
17208        assert_eq!(
17209            (kept.head.as_str(), kept.base.as_str()),
17210            (head.as_str(), base.as_str())
17211        );
17212        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17213    }
17214
17215    #[tokio::test]
17216    async fn review_only_summary_receives_the_configured_github_language() {
17217        let mut f = summary_fixture();
17218        f.runner.state.config.graph.github_language = "ja".into();
17219        let captured = f.repo.join("summary-prompt.txt");
17220        let author = &mut f.runner.state.config.agents[0];
17221        author.command = vec![
17222            "sh".into(),
17223            "-c".into(),
17224            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17225        ];
17226        author.env.insert(
17227            "PROMPT_FILE".into(),
17228            captured.to_string_lossy().into_owned(),
17229        );
17230        std::fs::write(
17231            &f.reply,
17232            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17233        )
17234        .unwrap();
17235        let m = posted(&mut f).await;
17236        let prompt = std::fs::read_to_string(captured).unwrap();
17237        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17238        assert!(!prompt.contains("in English"));
17239        assert_eq!(m.title, "feat(own): ファイルを追加");
17240        assert!(m.body.contains("own.txt を追加します。"));
17241    }
17242
17243    #[tokio::test]
17244    async fn review_fixes_come_after_the_whole_change_summary() {
17245        let mut f = summary_fixture();
17246        f.runner.state.reviews = vec![
17247            round_with_notes(1, Some("handled the PATH case")),
17248            round_with_notes(2, None),
17249        ];
17250        let m = posted(&mut f).await;
17251        let summary = m.body.find("## Summary").expect("summary");
17252        let fixes = m.body.find("## Review rounds").expect("review rounds");
17253        assert!(summary < fixes, "{}", m.body);
17254        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17255        assert!(m.body.contains("## Final verification"), "{}", m.body);
17256    }
17257
17258    #[tokio::test]
17259    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17260        let mut f = summary_fixture();
17261        let first = posted(&mut f).await;
17262        let first_head = f.runner.state.change_summary.clone().unwrap().head;
17263        // The same head reuses the record: a different reply is not read.
17264        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17265        let again = posted(&mut f).await;
17266        assert_eq!((again.title, again.body), (first.title, first.body));
17267
17268        let g = |args: &[&str]| {
17269            let out = std::process::Command::new("git")
17270                .args(args)
17271                .current_dir(&f.repo)
17272                .quiet()
17273                .output()
17274                .expect("spawn git");
17275            assert!(out.status.success(), "git {args:?}");
17276        };
17277        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17278        g(&["add", "-A"]);
17279        g(&["commit", "-q", "-m", "fix(own): handle more"]);
17280        let second = posted(&mut f).await;
17281        let kept = f.runner.state.change_summary.clone().unwrap();
17282        assert_ne!(kept.head, first_head);
17283        assert!(second.body.contains("Not read yet."), "{}", second.body);
17284        assert!(!second.body.contains("so the first summary is read"));
17285        assert_eq!(second.title, "feat(own): other");
17286    }
17287
17288    #[tokio::test]
17289    async fn an_unavailable_summary_is_stated_and_never_invented() {
17290        let mut f = summary_fixture();
17291        std::fs::write(&f.reply, "no title line here").unwrap();
17292        let m = posted(&mut f).await;
17293        assert!(
17294            m.body
17295                .contains("No description of what this change does could be generated"),
17296            "{}",
17297            m.body
17298        );
17299        assert!(m.body.contains("internal reports remain in local records"));
17300        assert!(f.runner.state.change_summary.is_none());
17301        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17302        // A preview asks nobody and claims nothing.
17303        let winner = f.runner.state.candidates[0].clone();
17304        let preview = f
17305            .runner
17306            .guarded_pr_message(&winner, None, false)
17307            .await
17308            .unwrap()
17309            .unwrap();
17310        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17311    }
17312
17313    #[test]
17314    fn pr_message_review_single_commit_uses_its_subject() {
17315        let state = review_state(&["feat(nats): per-role user"]);
17316        let m = pr_message(&state, 'A');
17317        assert_eq!(m.title, "feat(nats): per-role user");
17318        assert!(!m.body.contains("Review the work already"), "{}", m.body);
17319        assert!(m.body.contains("## Commits under review"), "{}", m.body);
17320    }
17321
17322    #[test]
17323    fn pr_message_review_multi_commit_takes_the_oldest() {
17324        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17325        let m = pr_message(&state, 'A');
17326        assert_eq!(m.title, "feat: the change");
17327        for s in ["feat: the change", "fix: typo", "fix: again"] {
17328            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17329        }
17330    }
17331
17332    #[test]
17333    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17334        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17335            let mut state = review_state(&[first, "fix: later fixup"]);
17336            state.candidates[0].branch = "機能/ログイン".to_owned();
17337            let m = pr_message(&state, 'A');
17338            assert!(
17339                m.title.starts_with("chore: land candidate A of run"),
17340                "{}",
17341                m.title
17342            );
17343        }
17344    }
17345
17346    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17347        BranchFacts {
17348            commits: commits
17349                .iter()
17350                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17351                .collect(),
17352            stat: stat.to_owned(),
17353        }
17354    }
17355
17356    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17357        let mut r = review_round(true, 0, 1, 1, true, true);
17358        r.round = round;
17359        r.fix = notes.map(|n| FixRecord {
17360            agent: "fixer".to_owned(),
17361            addressed: Vec::new(),
17362            rejected: Vec::new(),
17363            notes: n.to_owned(),
17364            committed: true,
17365            failed: None,
17366            duration_ms: 0,
17367            continuation: None,
17368        });
17369        r
17370    }
17371
17372    #[test]
17373    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17374        let state = review_state(&["ignored"]);
17375        let f = facts(
17376            &[
17377                (
17378                    "fix(login): resolve PATH on macOS",
17379                    "Login shells skip rc files.",
17380                ),
17381                ("fix: address review", ""),
17382            ],
17383            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17384        );
17385        let m = pr_message_with(&state, 'A', Some(&f));
17386        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17387        assert!(
17388            m.body
17389                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
17390            "{}",
17391            m.body
17392        );
17393        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17394        assert!(m.body.contains("## Diff stat"), "{}", m.body);
17395        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17396        for banned in [
17397            "Review the work already",
17398            "no task statement",
17399            "Original task",
17400        ] {
17401            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17402        }
17403        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17404    }
17405
17406    #[test]
17407    fn pr_message_review_truncates_a_huge_first_commit_body() {
17408        let state = review_state(&["ignored"]);
17409        let f = facts(
17410            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17411            "s",
17412        );
17413        let m = pr_message_with(&state, 'A', Some(&f));
17414        assert!(m.body.len() < 30_000, "{}", m.body.len());
17415        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17416        assert!(
17417            m.body.contains("- ... 1 more commit(s)"),
17418            "{}",
17419            m.body.len()
17420        );
17421        assert!(m.body.ends_with("magi:candidate-a\n"));
17422    }
17423
17424    #[test]
17425    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17426        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17427        assert_eq!(m.title, "feat: x");
17428        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17429        assert!(!m.body.contains("Diff stat"), "{}", m.body);
17430        assert!(!m.body.contains("no task statement"), "{}", m.body);
17431    }
17432
17433    #[test]
17434    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17435        let mut state = review_state(&["日本語の件名"]);
17436        state.candidates[0].branch = "fix/macos-login-path".to_owned();
17437        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17438        state.candidates[0].branch = "機能/ログイン".to_owned();
17439        assert!(
17440            pr_message(&state, 'A')
17441                .title
17442                .starts_with("chore: land candidate A")
17443        );
17444    }
17445
17446    #[test]
17447    fn pr_message_review_fixes_survive_a_clean_final_round() {
17448        let mut state = review_state(&["feat: x"]);
17449        state.reviews = vec![
17450            round_with_notes(1, Some("handled the PATH case")),
17451            round_with_notes(2, None),
17452        ];
17453        let body = pr_message(&state, 'A').body;
17454        assert!(
17455            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17456            "{body}"
17457        );
17458        assert!(body.contains("### Round 1"), "{body}");
17459
17460        state.reviews = vec![
17461            round_with_notes(1, Some("first fix")),
17462            round_with_notes(2, Some("")),
17463            round_with_notes(3, Some("second fix")),
17464            round_with_notes(4, None),
17465        ];
17466        let body = pr_message(&state, 'A').body;
17467        assert!(
17468            body.contains("### Round 1") && !body.contains("first fix"),
17469            "{body}"
17470        );
17471        assert!(
17472            body.contains("### Round 3") && !body.contains("second fix"),
17473            "{body}"
17474        );
17475        assert!(body.contains("### Round 2"), "{body}");
17476    }
17477
17478    const FINAL: &str = "abcdef1234567890";
17479
17480    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17481        let mut r = round_with_notes(round, None);
17482        r.verified_head = verified.map(str::to_owned);
17483        r
17484    }
17485
17486    fn ok_gate(state: &mut RunState, head: Option<&str>) {
17487        state.gate = vec![CommandOutcome {
17488            command: "gate".to_owned(),
17489            code: Some(0),
17490            output_tail: String::new(),
17491            duration_ms: 0,
17492            resource_blocked: false,
17493        }];
17494        state.gate_ran = true;
17495        state.gate_head = head.map(str::to_owned);
17496    }
17497
17498    fn body_at(state: &RunState) -> String {
17499        pr_message_raw(
17500            state,
17501            'A',
17502            None,
17503            crate::pr_outcome::FinalHead::At(FINAL),
17504            ChangeNote::NotAsked,
17505        )
17506        .body
17507    }
17508
17509    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17510
17511    #[test]
17512    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17513        let mut state = review_state(&["feat: x"]);
17514        state.reviews = vec![
17515            round_with_notes(1, Some(SANDBOXED)),
17516            passing_round(2, Some(FINAL)),
17517        ];
17518        ok_gate(&mut state, Some(FINAL));
17519        let body = body_at(&state);
17520        assert!(!body.contains("Handled the PATH case."), "{body}");
17521        assert!(!body.contains("could not commit"), "{body}");
17522        assert!(!body.contains("could not run"), "{body}");
17523        assert!(
17524            body.contains("The fix is committed on the branch"),
17525            "{body}"
17526        );
17527        assert!(!body.contains("sandbox"), "{body}");
17528        assert!(
17529            body.contains("Tests (`verify.e2e`): passed on the final commit"),
17530            "{body}"
17531        );
17532        assert!(
17533            body.contains("Gate (`verify.gate`): passed on the final commit"),
17534            "{body}"
17535        );
17536        // The raw report is untouched in the record.
17537        assert!(
17538            state.reviews[0]
17539                .fix
17540                .as_ref()
17541                .unwrap()
17542                .notes
17543                .contains("could not commit")
17544        );
17545    }
17546
17547    #[test]
17548    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17549        let mut state = review_state(&["feat: x"]);
17550        let mut r1 = round_with_notes(1, Some(SANDBOXED));
17551        r1.fix.as_mut().unwrap().committed = false;
17552        let mut r2 = passing_round(2, Some(FINAL));
17553        r2.e2e[0].code = Some(1);
17554        state.reviews = vec![r1, r2];
17555        let body = body_at(&state);
17556        assert!(!body.contains("could not commit"), "{body}");
17557        assert!(!body.contains("could not run"), "{body}");
17558        assert!(body.contains("This round produced no commit"), "{body}");
17559        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17560        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17561        assert!(!body.contains("passed"), "{body}");
17562    }
17563
17564    #[test]
17565    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17566        let mut state = review_state(&["feat: x"]);
17567        state.reviews = vec![
17568            round_with_notes(1, Some(SANDBOXED)),
17569            passing_round(2, Some("0123456789abcdef")),
17570        ];
17571        ok_gate(&mut state, Some("0123456789abcdef"));
17572        let body = body_at(&state);
17573        assert!(!body.contains("HTTP tests could not run"), "{body}");
17574        // The commit claim is still settled by the structured record.
17575        assert!(!body.contains("could not commit"), "{body}");
17576        assert!(
17577            body.contains("passed on an earlier commit (`0123456`)"),
17578            "{body}"
17579        );
17580        assert!(!body.contains("passed on the final commit"), "{body}");
17581
17582        // A gate recorded before `gate_head` existed is "not recorded".
17583        ok_gate(&mut state, None);
17584        let body = body_at(&state);
17585        assert!(
17586            body.contains("passed on a commit that was not recorded"),
17587            "{body}"
17588        );
17589    }
17590
17591    #[test]
17592    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17593        let state = state_with_summary(
17594            "add retries to the client",
17595            "TITLE: feat: retries\n\nDid it.",
17596        );
17597        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17598        assert_eq!(m.title, "feat: retries");
17599        assert!(
17600            m.body.contains("<summary>Original task</summary>"),
17601            "{}",
17602            m.body
17603        );
17604        assert!(!m.body.contains("Commits under review"), "{}", m.body);
17605        assert!(
17606            m.body
17607                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17608            "{}",
17609            m.body
17610        );
17611    }
17612
17613    #[test]
17614    fn pr_message_review_bounds_a_long_english_subject() {
17615        let long = format!("feat: {}", "word ".repeat(100));
17616        let m = pr_message(&review_state(&[&long]), 'A');
17617        assert!(m.title.starts_with("feat: word"), "{}", m.title);
17618        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17619    }
17620
17621    #[test]
17622    fn pr_message_implementation_run_is_unchanged_by_review_support() {
17623        let state = state_with_summary("add retries\n\ndetails", "- did some things");
17624        let m = pr_message(&state, 'A');
17625        assert_eq!(m.title, "add retries");
17626        assert!(m.body.contains("<summary>Original task</summary>"));
17627        assert!(!m.body.contains("Commits under review"));
17628        assert_eq!(landing_subject_source(&state), state.instruction);
17629    }
17630
17631    #[test]
17632    fn review_run_squash_subject_is_the_change_not_the_prompt() {
17633        let state = review_state(&["feat: the change", "fix: typo"]);
17634        let source = landing_subject_source(&state);
17635        assert_eq!(land::merge_subject("", &source), "feat: the change");
17636        assert_eq!(
17637            land::merge_subject("magi: candidate A (uncommitted work)", &source),
17638            "feat: the change"
17639        );
17640        // An operator's rename still wins.
17641        assert_eq!(
17642            land::merge_subject("feat: renamed by hand", &source),
17643            "feat: renamed by hand"
17644        );
17645        let blank = review_state(&["日本語"]);
17646        assert!(
17647            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17648        );
17649    }
17650
17651    #[test]
17652    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17653        let state = review_state(&["feat: the change"]);
17654        let source = landing_subject_source(&state);
17655        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17656        assert_eq!(
17657            land::merge_subject(landing_title(&state, old), &source),
17658            "feat: the change"
17659        );
17660        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17661        let task = state_with_summary("add retries", "");
17662        assert_eq!(landing_title(&task, old), old);
17663    }
17664
17665    #[test]
17666    fn pr_message_describes_the_change_not_the_task() {
17667        let state = state_with_summary(
17668            "今回やってほしいこと: results projector を直す",
17669            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17670        );
17671        let m = pr_message(&state, 'A');
17672        assert_eq!(m.title, "fix(web): batch the runs list reads");
17673        assert!(
17674            m.body.starts_with("## Summary\n\n- reads run.json once"),
17675            "{}",
17676            m.body
17677        );
17678        assert!(!m.body.contains("TITLE:"), "{}", m.body);
17679        let task_at = m.body.find("今回やってほしいこと").unwrap();
17680        let details_at = m.body.find("<details>").unwrap();
17681        assert!(
17682            details_at < task_at,
17683            "the task lives inside <details>: {}",
17684            m.body
17685        );
17686        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17687        assert!(m.body.contains("magi:candidate-a"));
17688    }
17689
17690    #[test]
17691    fn pr_message_falls_back_to_the_task_without_a_title_line() {
17692        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17693        let m = pr_message(&state, 'A');
17694        assert_eq!(m.title, "add retries");
17695        assert!(
17696            m.body.contains("## Summary\n\n- did some things"),
17697            "{}",
17698            m.body
17699        );
17700
17701        let none = RunState::new(
17702            PathBuf::from("/repo"),
17703            "main".to_owned(),
17704            "abc1234".to_owned(),
17705            "add retries".to_owned(),
17706            Config::default(),
17707        );
17708        let m = pr_message(&none, 'A');
17709        assert_eq!(m.title, "add retries");
17710        assert!(!m.body.contains("## Summary"), "{}", m.body);
17711    }
17712
17713    #[test]
17714    fn pr_message_refuses_the_candidate_commit_subject() {
17715        for bad in [
17716            "TITLE: magi: candidate A (uncommitted work)",
17717            "TITLE: chore: stuff (uncommitted work)",
17718            "TITLE:   ",
17719        ] {
17720            let state = state_with_summary("add retries", bad);
17721            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17722        }
17723    }
17724
17725    #[test]
17726    fn pr_message_bounds_a_very_long_task_and_title() {
17727        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17728        let state = state_with_summary(&long, "- nothing");
17729        let m = pr_message(&state, 'A');
17730        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17731        assert!(!m.title.contains('\n'));
17732
17733        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17734        let m = pr_message(&state, 'A');
17735        assert!(m.title.starts_with("feat: "));
17736        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17737        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17738    }
17739
17740    fn long_title_of(instruction: &str) -> String {
17741        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17742    }
17743
17744    #[test]
17745    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17746        let first = "Make the landing path keep a readable title for long tasks";
17747        let line = format!(
17748            "{first}. {}",
17749            "And then keep going with more words ".repeat(20)
17750        );
17751        let t = long_title_of(&line);
17752        assert_eq!(t, first);
17753        assert!(!t.starts_with("chore: land"));
17754    }
17755
17756    #[test]
17757    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17758        let line = "word ".repeat(200);
17759        let t = long_title_of(&line);
17760        assert!(t.ends_with("word..."), "{t}");
17761        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17762    }
17763
17764    #[test]
17765    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17766        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17767            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17768        }
17769    }
17770
17771    #[test]
17772    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17773        let mut state = state_with_summary("再試行を追加する", "- nothing");
17774        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17775        state.config.graph.github_language = "ja".into();
17776        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17777    }
17778
17779    #[test]
17780    fn pr_message_title_limit_is_exact() {
17781        let at = "a".repeat(PR_TITLE_MAX);
17782        assert_eq!(long_title_of(&at), at);
17783        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17784        assert!(over.ends_with("..."), "{over}");
17785        assert_eq!(over.chars().count(), PR_TITLE_MAX);
17786    }
17787
17788    #[test]
17789    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17790        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17791        let t = long_title_of(&line);
17792        assert!(t.ends_with("..."), "{t}");
17793        assert!(t.is_ascii(), "{t}");
17794    }
17795
17796    #[test]
17797    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17798        let line = format!(
17799            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17800            "plus more filler words ".repeat(20)
17801        );
17802        let t = long_title_of(&line);
17803        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17804    }
17805
17806    #[test]
17807    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17808        let line = format!("\n\n# {}", "title words ".repeat(40));
17809        let state = state_with_summary(&line, "- nothing");
17810        let m = pr_message(&state, 'A');
17811        assert!(m.title.starts_with("title words"), "{}", m.title);
17812        assert_eq!(
17813            land::merge_subject(&m.title, &landing_subject_source(&state)),
17814            m.title
17815        );
17816        // An operator's rename wins untouched.
17817        assert_eq!(
17818            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17819            "feat: renamed by hand"
17820        );
17821    }
17822
17823    #[tokio::test]
17824    async fn github_text_rewrite_is_bounded_and_falls_back() {
17825        crate::run::pin_test_home();
17826        for (reply, accepted) in [
17827            (
17828                "TITLE: fix: retries\nAdd retries for failed requests.",
17829                true,
17830            ),
17831            (
17832                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17833                false,
17834            ),
17835            (
17836                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17837                true,
17838            ),
17839            (
17840                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17841                true,
17842            ),
17843            (
17844                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17845                true,
17846            ),
17847            (
17848                "TITLE: fix: retries\n日本語の説明です。 token=secret",
17849                false,
17850            ),
17851            ("TITLE: fix: retries\n", false),
17852        ] {
17853            let dir = tempfile::tempdir().unwrap();
17854            let mut runner = runner_at(RunStatus::Gating);
17855            runner.state = state_with_summary(
17856                "add retries",
17857                "TITLE: fix: retries\n日本語の説明を書きます。",
17858            );
17859            runner.state.repo = dir.path().to_owned();
17860            runner.state.candidates[0].worktree = dir.path().to_owned();
17861            let mut author = spec("alpha");
17862            author.command = vec![
17863                "sh".into(),
17864                "-c".into(),
17865                "printf '%s' \"$REWRITE_REPLY\"".into(),
17866            ];
17867            author.env.insert("REWRITE_REPLY".into(), reply.into());
17868            runner.state.config.agents = vec![author];
17869            seed_public_summary(&mut runner);
17870            let winner = runner.state.candidates[0].clone();
17871            let first = runner
17872                .guarded_pr_message(&winner, None, true)
17873                .await
17874                .unwrap();
17875            let message = if accepted {
17876                first.expect("a passing rewrite never asks")
17877            } else {
17878                // Withheld: the run parks on a question and posts nothing yet.
17879                assert!(first.is_none());
17880                assert!(runner.state.parked);
17881                let store = ask::Questions::open();
17882                let asked: Vec<_> = store
17883                    .list()
17884                    .into_iter()
17885                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17886                    .collect();
17887                assert_eq!(asked.len(), 1);
17888                assert!(!asked[0].detail.contains("secret"));
17889                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17890                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17891                assert_eq!(
17892                    asked[0]
17893                        .choices
17894                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
17895                    rewrite.is_some()
17896                );
17897                // Silence: the question lapses and the neutral text is used,
17898                // without a second rewrite call or a second question.
17899                store
17900                    .update(&asked[0].id, |q| {
17901                        q.abandon("test");
17902                        Ok(())
17903                    })
17904                    .unwrap();
17905                let again = runner
17906                    .guarded_pr_message(&winner, None, true)
17907                    .await
17908                    .unwrap()
17909                    .expect("an abandoned question falls back");
17910                let n = store
17911                    .list()
17912                    .into_iter()
17913                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17914                    .count();
17915                assert_eq!(n, 1, "asked once per rejected text");
17916                again
17917            };
17918            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17919            assert!(!message.body.contains("secret"));
17920            assert!(!message.body.contains("prefix"));
17921            assert!(!message.title.contains("secret"));
17922            assert_eq!(
17923                message.body.contains("Add retries for failed requests."),
17924                accepted
17925            );
17926            assert_eq!(
17927                runner.state.seats["github-text-rewrite"].turns, 1,
17928                "only one rewrite invocation"
17929            );
17930            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17931            if accepted {
17932                assert!(runner.state.github_text.is_none());
17933                assert!(!runner.state.parked);
17934                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17935                assert!(message.body.contains("## Final verification"));
17936                assert!(message.body.contains("Evidence: final diff"));
17937                assert!(
17938                    message
17939                        .body
17940                        .contains(&format!("magi:run/{}", runner.state.id))
17941                );
17942            }
17943            if !accepted {
17944                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17945                assert!(
17946                    message
17947                        .body
17948                        .contains(&format!("magi:run/{}", runner.state.id))
17949                );
17950            }
17951        }
17952    }
17953
17954    fn scripted(id: &str, reply: &str) -> AgentSpec {
17955        let mut a = spec(id);
17956        a.command = vec![
17957            "sh".into(),
17958            "-c".into(),
17959            "printf '%s' \"$REWRITE_REPLY\"".into(),
17960        ];
17961        a.env.insert("REWRITE_REPLY".into(), reply.into());
17962        a
17963    }
17964
17965    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17966    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17967
17968    /// Gate tests inject generated public output, not internal reports.
17969    fn seed_public_summary(runner: &mut Runner) {
17970        let repo = &runner.state.repo;
17971        let g = |args: &[&str]| {
17972            let out = std::process::Command::new("git")
17973                .args(args)
17974                .current_dir(repo)
17975                .quiet()
17976                .output()
17977                .unwrap();
17978            assert!(out.status.success(), "git {args:?}");
17979            String::from_utf8_lossy(&out.stdout).trim().to_owned()
17980        };
17981        g(&["init", "-q", "-b", "main"]);
17982        g(&[
17983            "-c",
17984            "user.name=Test",
17985            "-c",
17986            "user.email=test@example.invalid",
17987            "commit",
17988            "-q",
17989            "--allow-empty",
17990            "-m",
17991            "feat: fixture",
17992        ]);
17993        let head = g(&["rev-parse", "HEAD"]);
17994        g(&["update-ref", "refs/remotes/origin/main", &head]);
17995        let evidence = crate::pr_outcome::summary_evidence(
17996            &runner.state,
17997            crate::pr_outcome::FinalHead::At(&head),
17998        );
17999        let generated = &runner.state.candidates[0].summary;
18000        runner.state.change_summary = Some(crate::run::ChangeSummary {
18001            base: head.clone(),
18002            head,
18003            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18004            title: summary_title(generated).unwrap(),
18005            body: summary_without_title(generated),
18006        });
18007    }
18008
18009    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18010        let mut runner = runner_at(RunStatus::Gating);
18011        runner.state = state_with_summary("add retries", JA_SUMMARY);
18012        if review_only {
18013            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18014        }
18015        runner.state.repo = dir.to_owned();
18016        runner.state.candidates[0].worktree = dir.to_owned();
18017        runner.state.config.agents = agents;
18018        seed_public_summary(&mut runner);
18019        runner
18020    }
18021
18022    fn gate_questions(runner: &Runner) -> usize {
18023        ask::Questions::open()
18024            .list()
18025            .into_iter()
18026            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18027            .count()
18028    }
18029
18030    #[tokio::test]
18031    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18032        crate::run::pin_test_home();
18033        let dir = tempfile::tempdir().unwrap();
18034        let mut runner = rewrite_runner(
18035            dir.path(),
18036            true,
18037            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18038        );
18039        runner.state.config.roles.language_judge =
18040            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18041        let winner = runner.state.candidates[0].clone();
18042        let msg = runner
18043            .guarded_pr_message(&winner, None, true)
18044            .await
18045            .unwrap()
18046            .expect("an English rewrite posts without asking");
18047        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18048        assert!(msg.body.contains("<details>kept</details>"));
18049        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18050        assert_eq!(gate_questions(&runner), 0);
18051        assert!(runner.state.github_text.is_none());
18052        assert!(!runner.state.seats.contains_key("impl-A"));
18053        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18054    }
18055
18056    #[tokio::test]
18057    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18058        crate::run::pin_test_home();
18059        let dir = tempfile::tempdir().unwrap();
18060        let mut runner = rewrite_runner(
18061            dir.path(),
18062            false,
18063            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18064        );
18065        runner.state.config.roles.language_judge =
18066            Some(crate::config::AgentChoice::One("beta".into()));
18067        runner.state.seats.insert(
18068            "impl-A".into(),
18069            SeatState::new("impl-A", "alpha", runner.state.seed),
18070        );
18071        let winner = runner.state.candidates[0].clone();
18072        let msg = runner
18073            .guarded_pr_message(&winner, None, true)
18074            .await
18075            .unwrap()
18076            .expect("rewritten");
18077        assert!(msg.body.contains("<details>kept</details>"));
18078        assert_eq!(runner.state.seats["impl-A"].turns, 0);
18079        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18080        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18081    }
18082
18083    #[tokio::test]
18084    async fn rewrite_reuses_a_resumable_implementer_session() {
18085        crate::run::pin_test_home();
18086        let dir = tempfile::tempdir().unwrap();
18087        let mut runner = rewrite_runner(
18088            dir.path(),
18089            false,
18090            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18091        );
18092        runner.state.config.roles.language_judge =
18093            Some(crate::config::AgentChoice::One("beta".into()));
18094        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18095        seat.turns = 1;
18096        runner.state.seats.insert("impl-A".into(), seat);
18097        let winner = runner.state.candidates[0].clone();
18098        runner
18099            .guarded_pr_message(&winner, None, true)
18100            .await
18101            .unwrap()
18102            .expect("rewritten");
18103        assert_eq!(runner.state.seats["impl-A"].turns, 2);
18104        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18105    }
18106
18107    #[tokio::test]
18108    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18109        crate::run::pin_test_home();
18110        let dir = tempfile::tempdir().unwrap();
18111        let mut broken = spec("bad");
18112        broken.command = vec!["false".into()];
18113        let mut runner = rewrite_runner(
18114            dir.path(),
18115            true,
18116            vec![
18117                broken,
18118                scripted("noformat", "Add retries."),
18119                scripted("good", EN_REWRITE),
18120            ],
18121        );
18122        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18123            "bad".into(),
18124            "noformat".into(),
18125            "good".into(),
18126        ]));
18127        let winner = runner.state.candidates[0].clone();
18128        let msg = runner
18129            .guarded_pr_message(&winner, None, true)
18130            .await
18131            .unwrap()
18132            .expect("the third agent rewrites");
18133        assert!(msg.body.contains("`src/net.rs`"));
18134        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18135        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18136    }
18137
18138    #[tokio::test]
18139    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18140        crate::run::pin_test_home();
18141        // (agents, role): still Japanese; no TITLE; nothing resolvable.
18142        for (agents, role, saved) in [
18143            (
18144                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18145                Some("beta"),
18146                true,
18147            ),
18148            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18149            (Vec::new(), None, false),
18150        ] {
18151            let dir = tempfile::tempdir().unwrap();
18152            let mut runner = rewrite_runner(dir.path(), true, agents);
18153            runner.state.config.roles.language_judge =
18154                role.map(|r| crate::config::AgentChoice::One(r.into()));
18155            let winner = runner.state.candidates[0].clone();
18156            let first = runner
18157                .guarded_pr_message(&winner, None, true)
18158                .await
18159                .unwrap();
18160            assert!(first.is_none(), "withheld text parks the run");
18161            assert!(runner.state.parked);
18162            assert_eq!(gate_questions(&runner), 1);
18163            let g = runner.state.github_text.as_ref().unwrap();
18164            assert_eq!(g.rewrite.is_some(), saved);
18165            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18166        }
18167    }
18168
18169    #[tokio::test]
18170    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18171        crate::run::pin_test_home();
18172        let dir = tempfile::tempdir().unwrap();
18173        let mut runner = runner_at(RunStatus::Gating);
18174        runner.state =
18175            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18176        runner.state.repo = dir.path().to_owned();
18177        runner.state.candidates[0].worktree = dir.path().to_owned();
18178        let mut author = spec("alpha");
18179        author.command = vec![
18180            "sh".into(),
18181            "-c".into(),
18182            "printf '%s' \"$REWRITE_REPLY\"".into(),
18183        ];
18184        author.env.insert(
18185            "REWRITE_REPLY".into(),
18186            format!(
18187                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18188                "English sentence. ".repeat(600)
18189            ),
18190        );
18191        runner.state.config.agents = vec![author];
18192        seed_public_summary(&mut runner);
18193        let winner = runner.state.candidates[0].clone();
18194        assert!(
18195            runner
18196                .guarded_pr_message(&winner, None, true)
18197                .await
18198                .unwrap()
18199                .is_none()
18200        );
18201        let record = runner.state.github_text.clone().unwrap();
18202        let rewrite = record.rewrite.clone().unwrap();
18203        assert_eq!(rewrite.failed, ["body-language"]);
18204        assert!(
18205            rewrite
18206                .body
18207                .ends_with(&format!("magi:run/{}", runner.state.id))
18208        );
18209        let store = ask::Questions::open();
18210        let q = store.get(record.question.as_ref().unwrap()).unwrap();
18211        assert!(
18212            q.choices
18213                .contains(&crate::github_text::USE_REWRITE.to_owned())
18214        );
18215        for label in [
18216            "ORIGINAL",
18217            "FALLBACK",
18218            "REWRITE",
18219            "Rules still failed: body-language",
18220            "truncated",
18221        ] {
18222            assert!(q.detail.contains(label), "{label}");
18223        }
18224        let artifact = std::fs::read_to_string(crate::run::artifact_path(
18225            &runner.state,
18226            &crate::github_text::artifact_name(&record.fingerprint),
18227        ))
18228        .unwrap();
18229        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18230        // Restart from serialized state, then retry after a bad owner title.
18231        runner.state =
18232            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18233        store
18234            .update(&q.id, |q| {
18235                q.say("まだ日本語")?;
18236                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18237            })
18238            .unwrap();
18239        assert!(
18240            runner
18241                .guarded_pr_message(&winner, None, true)
18242                .await
18243                .unwrap()
18244                .is_none()
18245        );
18246        let retry = runner.state.github_text.as_ref().unwrap();
18247        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18248        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18249        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18250        assert!(
18251            q.posting_comparison
18252                .as_ref()
18253                .unwrap()
18254                .matches_detail(&q.detail)
18255        );
18256        store
18257            .update(&q.id, |q| {
18258                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18259            })
18260            .unwrap();
18261        let message = runner
18262            .guarded_pr_message(&winner, None, true)
18263            .await
18264            .unwrap()
18265            .unwrap();
18266        assert_eq!(message.title, rewrite.title);
18267        assert_eq!(message.body, rewrite.body);
18268        assert_eq!(
18269            runner.state.github_text.as_ref().unwrap().choice,
18270            Some(GithubTextChoice::Rewrite)
18271        );
18272        let again = runner
18273            .guarded_pr_message(&winner, None, true)
18274            .await
18275            .unwrap()
18276            .unwrap();
18277        assert_eq!(again.title, message.title);
18278        assert_eq!(again.body, message.body);
18279        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18280        assert!(
18281            !runner
18282                .state
18283                .events
18284                .iter()
18285                .any(|e| e.message.contains(&rewrite.title)
18286                    || e.message.contains("English sentence."))
18287        );
18288    }
18289
18290    #[tokio::test]
18291    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18292        crate::run::pin_test_home();
18293        let dir = tempfile::tempdir().unwrap();
18294        let mut runner = runner_at(RunStatus::Gating);
18295        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18296        runner.state.repo = dir.path().to_owned();
18297        runner.state.candidates[0].worktree = dir.path().to_owned();
18298        seed_public_summary(&mut runner);
18299        let winner = runner.state.candidates[0].clone();
18300        assert!(
18301            runner
18302                .guarded_pr_message(&winner, None, true)
18303                .await
18304                .unwrap()
18305                .is_none()
18306        );
18307        let record = runner.state.github_text.as_ref().unwrap();
18308        assert!(record.rewrite.is_none());
18309        // Records written before the additive field deserialize with no rewrite.
18310        let mut old = serde_json::to_value(record).unwrap();
18311        old.as_object_mut().unwrap().remove("rewrite");
18312        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18313        assert!(old.rewrite.is_none());
18314        let id = record.question.clone().unwrap();
18315        ask::Questions::open()
18316            .update(&id, |q| {
18317                q.choices.push(crate::github_text::USE_REWRITE.into());
18318                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18319            })
18320            .unwrap();
18321        let message = runner
18322            .guarded_pr_message(&winner, None, true)
18323            .await
18324            .unwrap()
18325            .unwrap();
18326        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18327        assert_eq!(
18328            runner.state.github_text.as_ref().unwrap().choice,
18329            Some(GithubTextChoice::Fallback)
18330        );
18331    }
18332
18333    #[tokio::test]
18334    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18335        crate::run::pin_test_home();
18336        let dir = tempfile::tempdir().unwrap();
18337        let mut runner = runner_at(RunStatus::Gating);
18338        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18339        runner.state.repo = dir.path().to_owned();
18340        runner.state.candidates[0].worktree = dir.path().to_owned();
18341        seed_public_summary(&mut runner);
18342        let winner = runner.state.candidates[0].clone();
18343        assert!(
18344            runner
18345                .guarded_pr_message(&winner, None, true)
18346                .await
18347                .unwrap()
18348                .is_none()
18349        );
18350        let store = ask::Questions::open();
18351        let find = |run: &str| {
18352            store
18353                .list()
18354                .into_iter()
18355                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18356                .unwrap()
18357        };
18358        let run = runner.state.id.clone();
18359        let q = find(&run);
18360        let fp = runner
18361            .state
18362            .github_text
18363            .as_ref()
18364            .unwrap()
18365            .fingerprint
18366            .clone();
18367        let name = crate::github_text::artifact_name(&fp);
18368        let path = crate::run::artifact_path(&runner.state, &name);
18369        assert!(q.detail.contains(&path.display().to_string()));
18370        assert!(q.detail.contains("title-language: title"));
18371        let saved = std::fs::read_to_string(&path).expect("artifact written");
18372        assert!(saved.contains("日本語のタイトル"));
18373        assert!(q.cwd.is_none());
18374        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18375        // A bad replacement is refused and asked about once more.
18376        let say = |id: &str, text: &str, choice: &str| {
18377            store
18378                .update(id, |q| {
18379                    q.thread.push(ask::Turn {
18380                        who: ask::Who::Operator,
18381                        body: text.to_owned(),
18382                        at: jiff::Timestamp::now(),
18383                        note: None,
18384                    });
18385                    q.answer(ask::Answer::Choice(choice.to_owned()))
18386                })
18387                .unwrap();
18388        };
18389        say(&q.id, "まだ日本語", "use my text");
18390        assert!(
18391            runner
18392                .guarded_pr_message(&winner, None, true)
18393                .await
18394                .unwrap()
18395                .is_none()
18396        );
18397        let second = find(&run);
18398        assert_ne!(second.id, q.id);
18399        assert!(second.detail.contains("did not pass"));
18400        // The retry still compares the original text; the rejected
18401        // replacement is named by category only.
18402        assert!(second.detail.contains("日本語のタイトル"));
18403        assert!(!second.detail.contains("まだ日本語"));
18404        assert!(second.detail.contains(&path.display().to_string()));
18405        say(&second.id, "fix: retry failed requests", "use my text");
18406        let message = runner
18407            .guarded_pr_message(&winner, None, true)
18408            .await
18409            .unwrap()
18410            .expect("a vetted replacement is posted");
18411        assert_eq!(message.title, "fix: retry failed requests");
18412        assert_eq!(
18413            runner
18414                .state
18415                .github_text
18416                .as_ref()
18417                .unwrap()
18418                .chosen_title
18419                .as_deref(),
18420            Some("fix: retry failed requests")
18421        );
18422        assert!(runner.state.github_text.as_ref().unwrap().resolved);
18423        // Resumed again: the saved decision is applied, nothing is re-asked.
18424        let again = runner
18425            .guarded_pr_message(&winner, None, true)
18426            .await
18427            .unwrap()
18428            .unwrap();
18429        assert_eq!(again.title, message.title);
18430        assert_eq!(
18431            store
18432                .list()
18433                .into_iter()
18434                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18435                .count(),
18436            2
18437        );
18438    }
18439
18440    #[test]
18441    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18442        // What magi itself writes stays English under any configured language,
18443        // so a future localisation of these headings fails here. (The agents'
18444        // own text is also checked by the posting gate.)
18445        let mut state = state_with_summary(
18446            "add retries",
18447            "TITLE: fix(web): batch reads\n- reads run.json once",
18448        );
18449        state.config.graph.language = "ja".to_owned();
18450        let m = pr_message(&state, 'A');
18451        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18452        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18453
18454        // The task is the operator's own text: it goes in untouched, and the
18455        // fallback title (no summary) may be in its language too.
18456        let task = "今回やってほしいこと: results projector を直す";
18457        let mut state = state_with_summary(task, "- no title line");
18458        state.config.graph.language = "ja".to_owned();
18459        let m = pr_message(&state, 'A');
18460        assert_eq!(
18461            m.title,
18462            format!("chore: land candidate A of run {}", state.id)
18463        );
18464        assert!(
18465            m.body.contains(&format!(
18466                "<summary>Original task</summary>\n\n{task}\n\n</details>"
18467            )),
18468            "{}",
18469            m.body
18470        );
18471    }
18472
18473    #[test]
18474    fn pr_message_scrubs_home_paths_and_addresses() {
18475        let state = state_with_summary(
18476            "fix it in /Users/someone/src/x",
18477            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18478        );
18479        let m = pr_message(&state, 'A');
18480        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18481            assert!(!m.body.contains(leak), "{}", m.body);
18482        }
18483        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18484    }
18485
18486    #[test]
18487    fn pr_message_survives_a_task_that_closes_details() {
18488        let state = state_with_summary("a </details> b", "TITLE: fix: x");
18489        let m = pr_message(&state, 'A');
18490        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18491    }
18492
18493    #[test]
18494    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18495        let cmd = manual_merge_command(
18496            MergeStyle::Squash,
18497            Path::new("/repo"),
18498            "b",
18499            "fix: \"quoted\" $(x) `y`\n\nbody",
18500        );
18501        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18502    }
18503
18504    #[test]
18505    fn manual_merge_command_matches_the_configured_style() {
18506        let repo = Path::new("/repo");
18507        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18508
18509        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18510        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18511
18512        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18513        assert_eq!(
18514            squash,
18515            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18516             \"Merge magi run 0832 (candidate A)\""
18517        );
18518
18519        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18520        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18521    }
18522
18523    #[test]
18524    fn a_nudge_gets_a_quarter_of_the_budget() {
18525        // The judge and implement budgets magi ships with.
18526        assert_eq!(retry_budget(secs(1200), true), secs(300));
18527        assert_eq!(retry_budget(secs(3600), true), secs(900));
18528    }
18529
18530    #[test]
18531    fn a_resent_prompt_keeps_the_whole_budget() {
18532        // The seat kept no context, so the retry is the original job again and
18533        // shortening it would only guarantee a second failure.
18534        assert_eq!(retry_budget(secs(1200), false), secs(1200));
18535        assert_eq!(retry_budget(secs(60), false), secs(60));
18536    }
18537
18538    #[test]
18539    fn the_floor_never_exceeds_the_original_budget() {
18540        // A short configured timeout must not be *raised* by the floor: the
18541        // operator asked for a bound, and a retry may not outlast the attempt
18542        // it is retrying.
18543        assert_eq!(retry_budget(secs(60), true), secs(60));
18544        assert_eq!(retry_budget(secs(480), true), secs(120));
18545        assert_eq!(retry_budget(secs(0), true), secs(0));
18546    }
18547
18548    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18549        agent::CommandEvidence {
18550            id: "item1".to_owned(),
18551            description: "cargo test".to_owned(),
18552            exit_code,
18553            result_summary: String::new(),
18554            source: "codex".to_owned(),
18555        }
18556    }
18557
18558    #[test]
18559    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18560        // No evidence is not the same fact as unconfirmed evidence: a
18561        // backend with no adapter, or a reply that ran no commands at all,
18562        // must not be misread as carrying a dangling job.
18563        assert!(!has_unconfirmed_command(&[]));
18564    }
18565
18566    #[test]
18567    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18568        // Deliberately not a check on the exit code's *value*: a fixer
18569        // legitimately runs something that fails mid-iteration before it
18570        // succeeds, and that must never by itself reopen a valid report.
18571        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18572        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18573        assert!(!has_unconfirmed_command(&[
18574            evidence(Some(0)),
18575            evidence(Some(101))
18576        ]));
18577    }
18578
18579    #[test]
18580    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18581        assert!(has_unconfirmed_command(&[
18582            evidence(Some(0)),
18583            evidence(None)
18584        ]));
18585    }
18586
18587    #[test]
18588    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18589        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18590        assert_eq!(
18591            verified_noop_claim(true, &[], text).as_deref(),
18592            Some("already fixed by b32cfc4, on main.")
18593        );
18594    }
18595
18596    #[test]
18597    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18598        // A timeout or a bad exit code reads as the ordinary loss it is,
18599        // whatever the reply's own prose claims.
18600        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18601        assert!(verified_noop_claim(false, &[], text).is_none());
18602    }
18603
18604    #[test]
18605    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18606        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18607        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18608        // A confirmed command alongside the marker is fine.
18609        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18610    }
18611
18612    #[test]
18613    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18614        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18615    }
18616
18617    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
18618    /// pair, in order, labelled A, B, C, ...
18619    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18620        runner.state.candidates = shape
18621            .iter()
18622            .enumerate()
18623            .map(|(i, &(empty, verified))| Candidate {
18624                index: i,
18625                label: (b'A' + i as u8) as char,
18626                agent: "sonnet".to_owned(),
18627                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18628                worktree: PathBuf::from(format!("/wt/{i}")),
18629                summary: String::new(),
18630                stat: String::new(),
18631                files: 0,
18632                commits: 0,
18633                empty,
18634                failed: None,
18635                verified_noop: verified.map(str::to_owned),
18636                duration_ms: 0,
18637                folded: false,
18638            })
18639            .collect();
18640    }
18641
18642    #[test]
18643    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18644        ask_test_home();
18645        let mut runner = runner_at(RunStatus::Implementing);
18646        set_candidates(
18647            &mut runner,
18648            &[
18649                (true, Some("already on main at b32cfc4")),
18650                (true, Some("same fix, see the existing test")),
18651            ],
18652        );
18653
18654        runner
18655            .after_implement()
18656            .expect("a verified no-op is not an error");
18657
18658        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18659    }
18660
18661    #[test]
18662    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18663        ask_test_home();
18664        let mut runner = runner_at(RunStatus::Implementing);
18665        // Candidate A declares a verified no-op; candidate B simply wrote
18666        // nothing and said nothing about why. One candidate's claim is not
18667        // the whole run's agreement.
18668        set_candidates(
18669            &mut runner,
18670            &[(true, Some("already on main at b32cfc4")), (true, None)],
18671        );
18672
18673        let err = runner
18674            .after_implement()
18675            .expect_err("an unverified empty candidate must still fail the run");
18676
18677        assert!(
18678            err.to_string().contains("no candidate produced a change"),
18679            "{err}"
18680        );
18681        assert_eq!(runner.state.status, RunStatus::Failed);
18682    }
18683
18684    #[test]
18685    fn after_implement_still_fails_an_ordinary_all_empty_run() {
18686        ask_test_home();
18687        let mut runner = runner_at(RunStatus::Implementing);
18688        set_candidates(&mut runner, &[(true, None), (true, None)]);
18689
18690        let err = runner
18691            .after_implement()
18692            .expect_err("no candidate declared anything; this is an ordinary failure");
18693
18694        assert!(
18695            err.to_string().contains("no candidate produced a change"),
18696            "{err}"
18697        );
18698        assert_eq!(runner.state.status, RunStatus::Failed);
18699    }
18700}
18701
18702/// A graft rollback whose restore could not be confirmed (see
18703/// [`Runner::graft`]).
18704#[derive(Debug)]
18705struct GraftRestoreFailed(String);
18706
18707impl std::fmt::Display for GraftRestoreFailed {
18708    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18709        write!(
18710            f,
18711            "could not restore the winner to its pre-graft commit: {}",
18712            self.0
18713        )
18714    }
18715}
18716
18717impl std::error::Error for GraftRestoreFailed {}
18718
18719/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
18720/// not the agent: they never draw on an agent's own timeout, and running out of
18721/// them never reads as an agent failure.
18722#[cfg(test)]
18723mod resource_wait_tests {
18724    use super::*;
18725    use std::collections::BTreeMap;
18726
18727    fn sh(id: &str, script: &str) -> AgentSpec {
18728        AgentSpec {
18729            id: id.to_owned(),
18730            kind: crate::config::AgentKind::Command,
18731            model: None,
18732            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18733            extra_args: Vec::new(),
18734            env: BTreeMap::new(),
18735            prompt_delivery: None,
18736        }
18737    }
18738
18739    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18740        SeatJob {
18741            seat: SeatState::new("impl-A", &spec.id, 1),
18742            spec,
18743            cwd: dir.to_path_buf(),
18744            prompt: "do it".to_owned(),
18745            timeout,
18746            allow_write: true,
18747            sessions: false,
18748            artifacts: dir.join("art"),
18749            stem: "impl-A".to_owned(),
18750            handover: None,
18751        }
18752    }
18753
18754    fn state(resource_wait_secs: u64) -> RunState {
18755        let mut config = Config::default();
18756        config.graph.timeout_resource_wait = resource_wait_secs;
18757        RunState::new(
18758            PathBuf::from("/repo"),
18759            "main".to_owned(),
18760            "abc1234".to_owned(),
18761            "task".to_owned(),
18762            config,
18763        )
18764    }
18765
18766    fn hold(cache: &Path) -> crate::cache::Guard {
18767        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18768        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18769            crate::cache::AcquireOutcome::Acquired(g) => g,
18770            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18771        }
18772    }
18773
18774    async fn run(
18775        jobs: Vec<SeatJob>,
18776        sem: Arc<Semaphore>,
18777        cache: Option<&Path>,
18778        st: &mut RunState,
18779    ) -> Vec<(usize, SeatState, AgentOutcome)> {
18780        let prompts = Prompts::default();
18781        let ctx = WaveCtx {
18782            run: "r",
18783            node: "implement",
18784            prompts: &prompts,
18785            cache,
18786            round: None,
18787            carry_seats: false,
18788            pauses: None,
18789        };
18790        wave(jobs, sem, &ctx, st, 0).await
18791    }
18792
18793    #[tokio::test]
18794    async fn verification_cache_wait_preserves_the_command_budget() {
18795        crate::run::pin_test_home();
18796        let tmp = tempfile::tempdir().unwrap();
18797        let cache = tmp.path().join("target");
18798        let guard = hold(&cache);
18799        let release = tokio::spawn(async move {
18800            tokio::time::sleep(Duration::from_millis(100)).await;
18801            guard.release();
18802        });
18803        let mut st = state(10);
18804        let budget = Duration::from_millis(50);
18805        let wait = repository_wait(&st, None);
18806        let (outcomes, _) = with_cache_lease(
18807            &mut st,
18808            Some(&cache),
18809            "gate",
18810            "gate",
18811            tmp.path(),
18812            "abc1234",
18813            budget,
18814            "test gate",
18815            &wait,
18816            |_, available| async move {
18817                assert_eq!(
18818                    available, budget,
18819                    "contention must not shorten verification"
18820                );
18821                (Vec::new(), false, Vec::new())
18822            },
18823        )
18824        .await
18825        .unwrap();
18826        release.await.unwrap();
18827        assert!(outcomes.is_empty(), "{outcomes:?}");
18828        assert!(
18829            st.events
18830                .iter()
18831                .any(|event| event.message.contains("waiting for"))
18832        );
18833        assert!(st.resource_wait.is_none());
18834        // The verification body no longer owns the lease after returning.
18835        hold(&cache).release();
18836    }
18837
18838    #[tokio::test]
18839    async fn verification_cache_wait_expires_without_starting_commands() {
18840        crate::run::pin_test_home();
18841        let tmp = tempfile::tempdir().unwrap();
18842        let cache = tmp.path().join("target");
18843        let _held = hold(&cache);
18844        let mut st = state(0);
18845        let wait = repository_wait(&st, None);
18846        let (outcomes, _) = with_cache_lease(
18847            &mut st,
18848            Some(&cache),
18849            "gate",
18850            "gate",
18851            tmp.path(),
18852            "abc1234",
18853            Duration::from_secs(60),
18854            "test gate",
18855            &wait,
18856            |_, _| async { panic!("commands must not start without the cache lease") },
18857        )
18858        .await
18859        .unwrap();
18860        assert_eq!(outcomes.len(), 1);
18861        assert!(outcomes[0].resource_blocked);
18862        assert!(st.resource_wait.is_none());
18863    }
18864
18865    #[tokio::test]
18866    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18867        crate::run::pin_test_home();
18868        let tmp = tempfile::tempdir().unwrap();
18869        let cache = tmp.path().join("target-a");
18870        let guard = hold(&cache);
18871        // Held for longer than the agent's whole 1s budget.
18872        tokio::spawn(async move {
18873            tokio::time::sleep(Duration::from_millis(1500)).await;
18874            guard.release();
18875        });
18876        let mut st = state(60);
18877        let results = run(
18878            vec![job(
18879                sh("alpha", "echo ok"),
18880                tmp.path(),
18881                Duration::from_secs(1),
18882            )],
18883            Arc::new(Semaphore::new(1)),
18884            Some(&cache),
18885            &mut st,
18886        )
18887        .await;
18888        assert!(
18889            matches!(results[0].2, AgentOutcome::Ok(_)),
18890            "the wait must not be charged to the agent"
18891        );
18892        assert!(st.resource_block.is_none());
18893        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18894        assert!(st.resource_wait.is_none() && st.active.is_empty());
18895    }
18896
18897    #[tokio::test]
18898    #[cfg(unix)]
18899    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18900        use std::os::fd::AsRawFd as _;
18901        crate::run::pin_test_home();
18902        let tmp = tempfile::tempdir().unwrap();
18903        crate::git::git(tmp.path(), &["init"]).await.unwrap();
18904        let common = tmp.path().join(".git");
18905        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18906        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18907        assert_eq!(
18908            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18909            0
18910        );
18911        let mut st = state(1);
18912        st.config.coordination = crate::config::test_coordination();
18913        let mut reader = job(
18914            sh("reader", "echo must-not-run"),
18915            tmp.path(),
18916            Duration::from_secs(30),
18917        );
18918        reader.allow_write = false;
18919        let results = run(
18920            vec![reader.clone()],
18921            Arc::new(Semaphore::new(1)),
18922            None,
18923            &mut st,
18924        )
18925        .await;
18926        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18927        assert_eq!(results[0].1.turns, 0);
18928        assert!(FailClass::of(&results[0].2).is_none());
18929        assert_eq!(
18930            st.resource_block.as_ref().unwrap().kind,
18931            "repository coordination"
18932        );
18933        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18934        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18935        drop(external);
18936        let editor = crate::repo_lock::SeatLease::for_checkout(
18937            tmp.path(),
18938            true,
18939            crate::config::test_coordination().as_ref(),
18940        )
18941        .await
18942        .unwrap();
18943        let pause = Pause::new();
18944        let interrupt = Pause::new();
18945        let prompts = Prompts::default();
18946        let ctx = WaveCtx {
18947            run: "r",
18948            node: "review",
18949            prompts: &prompts,
18950            cache: None,
18951            round: None,
18952            carry_seats: false,
18953            pauses: Some((&pause, &interrupt)),
18954        };
18955        let (results, ()) = tokio::join!(
18956            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18957            async {
18958                tokio::time::sleep(Duration::from_millis(50)).await;
18959                pause.park();
18960            }
18961        );
18962        assert!(
18963            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18964        );
18965        assert_eq!(results[0].1.turns, 0);
18966        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18967        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18968        drop(editor);
18969    }
18970
18971    #[tokio::test]
18972    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18973        crate::run::pin_test_home();
18974        let tmp = tempfile::tempdir().unwrap();
18975        let cache = tmp.path().join("target-b");
18976        let _held = hold(&cache);
18977        let mut st = state(1);
18978        let results = run(
18979            vec![job(
18980                sh("alpha", "echo ok"),
18981                tmp.path(),
18982                Duration::from_secs(1),
18983            )],
18984            Arc::new(Semaphore::new(1)),
18985            Some(&cache),
18986            &mut st,
18987        )
18988        .await;
18989        let (_, seat, out) = &results[0];
18990        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18991        assert!(wave_blocked(&results));
18992        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18993        assert_eq!(seat.turns, 0, "nothing was asked");
18994        assert!(
18995            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18996            "no agent was spawned"
18997        );
18998        assert!(FailClass::of(out).is_none(), "not a failure class");
18999        let block = st.resource_block.as_ref().expect("recorded for the resume");
19000        assert_eq!(block.kind, "shared build cache");
19001        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19002        assert!(st.active.is_empty());
19003    }
19004
19005    #[tokio::test]
19006    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19007        crate::run::pin_test_home();
19008        let tmp = tempfile::tempdir().unwrap();
19009        let mut st = state(1);
19010        let results = run(
19011            vec![job(
19012                sh("alpha", "echo ok"),
19013                tmp.path(),
19014                Duration::from_secs(1),
19015            )],
19016            Arc::new(Semaphore::new(0)),
19017            None,
19018            &mut st,
19019        )
19020        .await;
19021        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19022        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19023        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19024    }
19025
19026    #[tokio::test]
19027    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19028        crate::run::pin_test_home();
19029        let tmp = tempfile::tempdir().unwrap();
19030        let mut st = state(60);
19031        let results = run(
19032            vec![job(
19033                sh("alpha", "sleep 30"),
19034                tmp.path(),
19035                Duration::from_secs(1),
19036            )],
19037            Arc::new(Semaphore::new(1)),
19038            None,
19039            &mut st,
19040        )
19041        .await;
19042        let out = &results[0].2;
19043        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19044        let class = FailClass::of(out).expect("a failure");
19045        assert_eq!(class, FailClass::Timeout);
19046        assert!(should_hand_over(None, &class), "existing handover policy");
19047        assert!(st.resource_block.is_none());
19048    }
19049}