Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54    Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58    ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62/// How much verification output is kept and fed back to the fixer.
63const OUTPUT_TAIL: usize = 8_000;
64
65/// Bytes of a failing command's output kept in an event, so the reason a run
66/// stopped is readable from the report without opening `run.json`.
67const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
70/// command's pid before releasing the build cache's lease.
71const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
74/// command's pid to actually exit before giving up and releasing anyway.
75///
76/// A timeout means the process was asked to die (`kill_on_drop`,
77/// `start_kill`), not that it already has — on Windows in particular that can
78/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
79/// the instant the command returns would let the very next acquirer (this
80/// run's own next round, another run's verification, the janitor's prune)
81/// start touching the same directory while it might still be writing to it,
82/// so this polls the actual pid — real confirmation, not a fixed guess —
83/// until it is gone or this ceiling is reached. It is still not full
84/// process-tree reaping: a grandchild the timed-out process spawned and that
85/// outlives it independently is invisible to a pid check, and continuing to
86/// observe and collect *that* stays a different piece of work with its own
87/// owner. Set generously because the common case returns early the moment
88/// the pid is confirmed gone, not because every timeout pays this in full.
89const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91/// Consecutive review rounds with no tree progress (see
92/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
93/// instead of spending the rest of the round budget.
94///
95/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
96/// legitimately finds nothing left to change (its previous round's fix already
97/// covered it, and this round's reviewers re-raised only nits) looks the same
98/// as one that is spinning, for exactly one round. Two in a row is where the
99/// two stop being distinguishable, and a review round on this workload has
100/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
101/// third attempt at a tree that has not moved twice running is pure cost.
102/// This does not touch `review_rounds` itself, which stays the operator's
103/// call.
104pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
107/// a base that moved before giving up and leaving the run `Blocked` for a
108/// person.
109///
110/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
111/// that keeps moving faster than a run can catch it is not something more
112/// rebasing fixes, it is a person's call. Not the same *number as*
113/// `land_rounds` - this budget is spent before a pull request exists, land's
114/// after - but bounded for the identical reason, so it uses the same
115/// default. Counted across both call sites in [`Runner::finish_after_tally`]
116/// (once before review, once before the gate), because either one finding
117/// the base still moving is the same signal.
118const BASE_SYNC_ROUNDS: usize = 4;
119
120/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
121/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
122/// reply held no [`FixReport`].
123///
124/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
125/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
126/// "I'll pause here until the `cargo make check` background run reports
127/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
128/// No `FixReport` was ever collected from that seat, and the run moved on to
129/// the next review round regardless.
130///
131/// Bounded independently of `review_rounds` and `graph.retries`: this
132/// recovers one seat's missing report mid-round, not a new round of review or
133/// an ordinary parse retry, and must not itself become the unbounded wait the
134/// rest of this module exists to avoid.
135const MAX_FIX_CONTINUATIONS: usize = 2;
136
137/// One queued agent invocation.
138///
139/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
140/// CLI hung up on its own stream is asked again from the same job rather than
141/// rebuilt from scratch. See [`Runner::resume_undelivered`].
142#[derive(Clone)]
143struct SeatJob {
144    spec: AgentSpec,
145    seat: SeatState,
146    cwd: PathBuf,
147    prompt: String,
148    timeout: Duration,
149    allow_write: bool,
150    sessions: bool,
151    artifacts: PathBuf,
152    stem: String,
153    /// The prompt for a seat that has been handed to another roster agent
154    /// (a fresh session): everything the original seat would have
155    /// remembered. `None` when `prompt` already carries it, as the first
156    /// ranking and the implement prompt do. Never a resume-style prompt.
157    handover: Option<String>,
158}
159
160/// How the graph reads one agent invocation.
161///
162/// Quota is split out from an ordinary failure on purpose: a rate-limited call
163/// is known to fail again if retried now, so the retry loop must not spend an
164/// attempt on it. `Dropped` is split out for the opposite reason: unlike
165/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
166/// error JSON, never the agent's answer — a caller that matched only
167/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
168/// left to read that JSON as if it were usable output. `resume_undelivered`
169/// is the only caller that acts on it; everywhere else it is reported like an
170/// ordinary failure.
171enum AgentOutcome {
172    /// A usable output.
173    Ok(AgentOutput),
174    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
175    Quota(AgentOutput),
176    /// The CLI hung up on its own stream after billed work. See
177    /// [`agent::AgentOutput::work_undelivered`].
178    Dropped(AgentOutput),
179    /// Any other failure: a timeout, a bad exit code, an empty reply.
180    Failed(String),
181    /// A resource prevented admission or interrupted execution. Not an agent
182    /// failure: never advance a fallback chain or spend an attempt on it.
183    /// Post-spawn interruptions carry a typed error into wave, which records
184    /// implementation preservation before returning this outcome. This outcome
185    /// alone is not proof that no source changes were made.
186    /// [`RunState::resource_block`] says why.
187    ResourceBlocked(String),
188}
189
190/// A request to park the run at its next node boundary.
191///
192/// Cloning is how the request travels: the loop keeps one handle and hands a
193/// clone to each [`Runner`], and every clone points at the same flag. There
194/// is no channel because there is nothing to send - the only message is
195/// "park", it is idempotent, and a flag cannot be missed by a receiver that
196/// was not listening yet.
197///
198/// The boundary is what makes this cheap. Every node writes the run's state
199/// before the next one starts, and every node skips what is already recorded:
200/// `prep` returns early once candidates exist, `implement` asks only the seats
201/// with nothing on disk, `judge` returns early once judgements exist. So a
202/// parked run resumes into exactly the node it stopped before, and no agent
203/// work is thrown away. Killing the process mid-node, by contrast, loses
204/// whatever the seats in flight had not yet written - which for an implement
205/// wave is an hour of paid work.
206///
207/// A [`Runner`] watches two independent handles of this type - see
208/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
209/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
210/// clone covering the whole daemon's lifetime and is never asked to un-park,
211/// which is correct exactly because nothing is dispatched after it fires.
212/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
213/// that parks for an interrupted task must go on to run other tasks
214/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
215/// reusing the daemon-wide one.
216#[derive(Debug, Clone, Default)]
217pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
218
219impl Pause {
220    /// A pause nobody has asked for yet.
221    #[must_use]
222    pub fn new() -> Self {
223        Self::default()
224    }
225
226    /// Ask the run to park at its next node boundary. Idempotent.
227    pub fn park(&self) {
228        self.0.store(true, Ordering::SeqCst);
229    }
230
231    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
232    /// fold into the run's own `park` event - so an operator reading the run
233    /// later knows this was a deliberate interrupt rather than a shutdown or
234    /// a binary swap. The first reason recorded wins; a park already in
235    /// flight is not relabelled by a second, unrelated request.
236    pub fn park_because(&self, reason: impl Into<String>) {
237        self.park_for(ParkCause::Interrupt {
238            reason: reason.into(),
239        });
240    }
241
242    /// [`Pause::park`], recording the structured cause. The first cause
243    /// recorded wins, like [`Pause::park_because`].
244    pub fn park_for(&self, cause: ParkCause) {
245        let mut reason_guard = self
246            .1
247            .lock()
248            .unwrap_or_else(std::sync::PoisonError::into_inner);
249        if reason_guard.is_none() {
250            *reason_guard = Some(cause);
251        }
252        drop(reason_guard);
253        self.park();
254    }
255
256    /// Has a park been asked for?
257    #[must_use]
258    pub fn parked(&self) -> bool {
259        self.0.load(Ordering::SeqCst)
260    }
261
262    /// Why the park was asked for, when the caller used [`Pause::park_because`].
263    #[must_use]
264    pub fn reason(&self) -> Option<String> {
265        self.cause().map(|c| c.label())
266    }
267
268    /// The structured cause, when one was recorded.
269    #[must_use]
270    pub fn cause(&self) -> Option<ParkCause> {
271        self.1
272            .lock()
273            .unwrap_or_else(std::sync::PoisonError::into_inner)
274            .clone()
275    }
276}
277
278/// Drives one run.
279pub struct Runner {
280    /// Run state; public so the CLI can report on it.
281    pub state: RunState,
282    roles: ResolvedRoles,
283    sem: Arc<Semaphore>,
284    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
285    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
286    /// this is never the same handle as `interrupt`.
287    pause: Pause,
288    /// Set when `magi serve`'s interrupt scheduler wants this specific run
289    /// parked at its next node boundary, to let a task marked
290    /// [`crate::queue::Task::interrupt`] run alone before this one carries
291    /// on. Unlike `pause`, a fresh, unshared handle per run - see
292    /// [`Runner::watch_interrupt`].
293    interrupt: Pause,
294}
295
296/// Where the branch's own commits start: its merge base with the base branch
297/// as the remote has it now, else with the recorded `base_commit`. A branch
298/// rebased onto a base that moved past `base_commit` would otherwise count
299/// the base's commits as its own under `base_commit..branch`.
300async fn review_base(
301    repo: &Path,
302    remote: &str,
303    base_branch: &str,
304    base_commit: &str,
305    branch: &str,
306) -> String {
307    review_base_checked(repo, remote, base_branch, base_commit, branch)
308        .await
309        .0
310}
311
312/// [`review_base`] plus whether the base was read from a freshly fetched
313/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
314/// never older than `base_commit`'s view of the base), but the answer is then
315/// not trusted to rewrite a pull request's title.
316async fn review_base_checked(
317    repo: &Path,
318    remote: &str,
319    base_branch: &str,
320    base_commit: &str,
321    branch: &str,
322) -> (String, bool) {
323    let tracking = format!("{remote}/{base_branch}");
324    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
325    if git::rev_exists(repo, &tracking).await
326        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
327        && !mb.is_empty()
328    {
329        return (mb, fresh);
330    }
331    let mb = git::merge_base(repo, base_commit, branch)
332        .await
333        .ok()
334        .filter(|mb| !mb.is_empty())
335        .unwrap_or_else(|| base_commit.to_owned());
336    (mb, false)
337}
338
339/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
340/// when git cannot say or finds nothing: a stale list is better than a wrong
341/// or empty one.
342pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
343    if !is_review_run(state) {
344        return;
345    }
346    let base = review_base(
347        &state.repo,
348        &state.config.merge.remote,
349        &state.base_branch,
350        &state.base_commit,
351        branch,
352    )
353    .await;
354    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
355        && !subjects.is_empty()
356        && state.reviewed_commits.as_ref() != Some(&subjects)
357    {
358        state.reviewed_commits = Some(subjects);
359        state.save().ok();
360    }
361}
362
363/// Subjects of the base's commits between the recorded start and the branch's
364/// merge base: what a stale `base_commit..branch` would have mistaken for the
365/// branch's own work.
366async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
367    let (base, trusted) = review_base_checked(
368        &state.repo,
369        &state.config.merge.remote,
370        &state.base_branch,
371        &state.base_commit,
372        branch,
373    )
374    .await;
375    if !trusted {
376        return None;
377    }
378    git::subjects(&state.repo, &state.base_commit, &base)
379        .await
380        .ok()
381}
382
383/// May an adopted pull request's title be replaced with `computed`? Only when
384/// it is empty, magi's own shape, or a base commit's subject that leaked in;
385/// a title a person wrote stays. Never when `computed` is itself a leak.
386pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
387    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
388    if is_leak(computed) {
389        return false;
390    }
391    let cur = current.trim();
392    cur.is_empty()
393        || cur.starts_with(REVIEW_PROMPT_OPENING)
394        || cur.starts_with("chore: land candidate")
395        || cur.starts_with("magi: candidate")
396        || is_leak(cur)
397}
398
399/// The commit a run branches from: the base branch as the remote has it.
400///
401/// Two failures this replaces. A run used to branch off `HEAD` and so refused
402/// to start on a dirty tree, which made `magi serve` decline every task for as
403/// long as the operator had work in progress - most of the time. Branching off
404/// the *local* base branch fixed that and introduced a worse one: `land` merges
405/// the winner on GitHub, nothing updates the local ref, and the next run
406/// branches off a base missing everything the previous runs landed. Two tasks
407/// in a row from a phone would have had the second silently re-implementing
408/// against stale code and opening a pull request that reverted the first.
409///
410/// Only refs move here - no checkout, no local branch, no merge - so it is safe
411/// with uncommitted work in the tree. A machine with no network still starts:
412/// the fetch may fail and the local tip is used with a warning, because
413/// refusing to run offline is a worse failure than running against a base the
414/// operator can see for themselves.
415///
416/// One function, called by both entry points. Two answers to "where does a run
417/// branch from" is the kind of drift nobody notices until a diff is wrong.
418/// Bring the local `branch` in line with `<remote>/<branch>` before a review
419/// checks it out.
420///
421/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
422/// anything other than plain `git push` from this checkout (a jj colocated
423/// workspace, another clone) moves only the remote-tracking ref - so the local
424/// one can be a stale placeholder. It moves only when local is behind the remote or is an
425/// empty placeholder that diverged from it; unpushed local work is kept, and a real
426/// divergence is refused rather than guessed at.
427async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
428    let tracking = format!("{remote}/{branch}");
429    let fetched = git::fetch(repo, remote, branch).await;
430    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
431    let local_exists = git::branch_exists(repo, branch).await?;
432    if !fresh {
433        if !local_exists {
434            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
435        }
436        tracing::warn!(
437            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
438        );
439        return Ok(());
440    }
441    let remote_sha = git::rev_parse(repo, &tracking).await?;
442    if !local_exists {
443        git::git(repo, &["branch", branch, &tracking]).await?;
444        return Ok(());
445    }
446    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
447    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
448        return Ok(());
449    }
450    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
451        // Diverged. `reconcile` settles it only when it can prove nothing is
452        // lost: a local tip that is the remote's change rebased is pushed over
453        // it (lease pinned to the tip read here), a tip whose every extra
454        // commit is empty is a placeholder the remote's work replaced, and
455        // anything else is two different changes - a question for a person.
456        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
457            .await?
458        {
459            crate::reconcile::Reconciliation::Pushed => {
460                tracing::warn!(
461                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
462                    short(&local_sha),
463                    short(&remote_sha)
464                );
465                return Ok(());
466            }
467            crate::reconcile::Reconciliation::Placeholder => {}
468            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
469        }
470    }
471    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
472    if !out.ok() {
473        bail!(
474            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
475            short(&local_sha),
476            short(&remote_sha),
477            out.stderr
478        );
479    }
480    tracing::warn!(
481        "local `{branch}` was stale: fast-forwarded {} -> {}",
482        short(&local_sha),
483        short(&remote_sha)
484    );
485    Ok(())
486}
487
488async fn resolve_base(
489    repo: &Path,
490    base_branch: &str,
491    remote: &str,
492    coordination: Option<&crate::config::Coordination>,
493) -> Result<crate::run::FetchedBase> {
494    let _lease = crate::repo_lock::SeatLease::operation_with(
495        repo,
496        coordination,
497        &crate::repo_lock::WaitPolicy::current(),
498    )
499    .await?;
500    let tracking = format!("{remote}/{base_branch}");
501    let fetched = git::fetch(repo, remote, base_branch).await;
502    if let Ok(out) = &fetched
503        && out.ok()
504        && git::rev_exists(repo, &tracking).await
505    {
506        return Ok(crate::run::FetchedBase {
507            remote: remote.to_owned(),
508            branch: base_branch.to_owned(),
509            commit: git::rev_parse(repo, &tracking).await?,
510            fetched_at: jiff::Timestamp::now(),
511        });
512    }
513    let why = match &fetched {
514        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
515        Ok(_) => format!("{remote} has no {base_branch}"),
516        Err(e) => e.to_string(),
517    };
518    // No fallback to the local branch: it may be behind, and branching off an
519    // old commit is the stale-checkout bug this check exists to prevent.
520    bail!(
521        "cannot read {tracking} ({why}); refusing to branch off the local \
522         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
523         base / remote in magi.toml. This is launcher preparation: check the \
524         launcher's Git administration access, network and repository coordination \
525         before retrying; an unattended seat cannot approve those permissions"
526    )
527}
528
529/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
530/// on an early return or a panic.
531///
532/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
533/// manual `magi fix` invocations against the same run are otherwise
534/// invisible to each other and would race to remove and recreate the same
535/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
536/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
537/// which is only ever reclaimed later, out of band, by
538/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
539/// `magi fix` invocation is not necessarily running under either of those, so
540/// nothing would ever sweep a lock a killed or crashed process left behind.
541/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
542/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
543/// lease use: an unreadable or unparsable pid, or a liveness query the
544/// platform cannot answer, reads as alive and the lock is left in place.
545struct FixClaim {
546    path: PathBuf,
547}
548
549impl FixClaim {
550    fn acquire(dir: &Path) -> Result<Self> {
551        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
552        let path = dir.join("fix.lock");
553        match Self::create(&path) {
554            Ok(claim) => Ok(claim),
555            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
556                if Self::reclaim_if_dead(&path) {
557                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
558                } else {
559                    bail!(
560                        "another repair or `magi fix` is already running for this run ({} exists)",
561                        path.display()
562                    )
563                }
564            }
565            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
566        }
567    }
568
569    fn create(path: &Path) -> std::io::Result<Self> {
570        let mut f = std::fs::OpenOptions::new()
571            .write(true)
572            .create_new(true)
573            .open(path)?;
574        use std::io::Write as _;
575        // Read back by `reclaim_if_dead` on a later, stuck invocation.
576        writeln!(f, "{}", std::process::id())?;
577        Ok(Self {
578            path: path.to_owned(),
579        })
580    }
581
582    /// True if the lock named a process confirmed dead, in which case it was
583    /// also removed. Never true on an unreadable file, an unparsable pid, or
584    /// a liveness query the platform cannot answer — see this type's own doc.
585    fn reclaim_if_dead(path: &Path) -> bool {
586        let dead = std::fs::read_to_string(path)
587            .ok()
588            .and_then(|body| body.trim().parse::<u32>().ok())
589            .is_some_and(|pid| !crate::proc::pid_alive(pid));
590        dead && std::fs::remove_file(path).is_ok()
591    }
592}
593
594impl Drop for FixClaim {
595    fn drop(&mut self) {
596        let _ = std::fs::remove_file(&self.path);
597    }
598}
599
600/// Holds the mutable runner as well as its durable lease: cancellation must
601/// clear both copies of ownership, including state a caller may save later.
602struct DriverExecution<'a> {
603    runner: &'a mut Runner,
604    driver: crate::driver::DriverGuard,
605}
606
607impl Drop for DriverExecution<'_> {
608    fn drop(&mut self) {
609        self.driver.release_local(&mut self.runner.state);
610    }
611}
612
613impl Runner {
614    /// Start a fresh run against `repo`.
615    pub async fn start(
616        repo: &Path,
617        instruction: String,
618        config: Config,
619        origin: Origin,
620    ) -> Result<Self> {
621        Self::start_naming(repo, instruction, "", config, origin).await
622    }
623
624    /// [`Runner::start`] for a queued task: `also_scan` (the task's title) is
625    /// searched for branch and commit references along with the instruction,
626    /// since a task may name the work it is about only in its title.
627    pub async fn start_naming(
628        repo: &Path,
629        instruction: String,
630        also_scan: &str,
631        config: Config,
632        origin: Origin,
633    ) -> Result<Self> {
634        let repo = git::toplevel(repo).await?;
635        let missing = agent::missing_programs(&config.agents);
636        if !missing.is_empty() {
637            bail!(
638                "these agent programs are not on PATH: {}. Fix the roster in \
639                 magi.toml or install them.",
640                missing.join(", ")
641            );
642        }
643        let base_branch =
644            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
645                .await?;
646        let fetched_base = resolve_base(
647            &repo,
648            &base_branch,
649            &config.merge.remote,
650            config.coordination.as_ref(),
651        )
652        .await?;
653        let base_commit = fetched_base.commit.clone();
654        let roles = config.resolve_roles()?;
655        let max_parallel = config.graph.max_parallel.max(1);
656        // A task that points at work already in the repository starts from
657        // it; what the repository says about each reference is recorded.
658        let seeds = refs::resolve(
659            &repo,
660            &base_commit,
661            &config.merge.remote,
662            &format!("{also_scan}\n{instruction}"),
663        )
664        .await;
665        refs::plan(&repo, &seeds).await?;
666        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
667        // Recorded before the first save, so a crash right after minting
668        // cannot leave a run with no origin. Legibility only: nothing reads it
669        // to decide anything.
670        state.fetched_base = Some(fetched_base);
671        state.origin = Some(origin);
672        for seed in &seeds {
673            state.event(
674                "seed",
675                refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
676            );
677        }
678        state.seeds = seeds;
679        state.event("start", format!("run {} created", state.id));
680        state.save()?;
681        Ok(Self {
682            state,
683            roles,
684            sem: Arc::new(Semaphore::new(max_parallel)),
685            pause: Pause::new(),
686            interrupt: Pause::new(),
687        })
688    }
689
690    /// Open a review-only run against work that already exists on `branch`.
691    ///
692    /// The expensive half of the graph is the implement wave — measured at
693    /// 111 and 134 internal tool-loop turns on this repository, against a
694    /// handful for a judge or a reviewer. The cheap half is worth running on
695    /// hand-written work too, and there was no way to reach it.
696    ///
697    /// No new state and no schema change are needed: a run with **one** viable
698    /// candidate and a tally already decided degrades `execute` to exactly
699    /// review → gate → merge, because `judge` skips a single-candidate field,
700    /// `deliberate` has fewer than two first choices to reconcile, `vote`
701    /// returns early, `tally` is already present and `fold_losers` has no
702    /// losers. Resuming such a run therefore does the right thing as well.
703    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
704        Self::review_taking_over(repo, branch, config, None, origin).await
705    }
706
707    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
708    /// at the same task still has `branch` checked out, its worktree is
709    /// released first if that is safe (see [`crate::handover`]), and the
710    /// review refuses with the reason if it is not. `None` is a hand-run
711    /// review: it has no earlier attempts, so only a worktree of a dead run
712    /// magi recorded itself can be released.
713    pub async fn review_taking_over(
714        repo: &Path,
715        branch: &str,
716        config: Config,
717        takeover: Option<crate::handover::Takeover>,
718        origin: Origin,
719    ) -> Result<Self> {
720        let repo = git::toplevel(repo).await?;
721        let missing = agent::missing_programs(&config.agents);
722        if !missing.is_empty() {
723            bail!(
724                "these agent programs are not on PATH: {}. Fix the roster in \
725                 magi.toml or install them.",
726                missing.join(", ")
727            );
728        }
729        let base_branch =
730            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
731                .await?;
732        if base_branch == branch {
733            bail!("`{branch}` is the base branch; there is nothing to review against");
734        }
735        let fetched_base = resolve_base(
736            &repo,
737            &base_branch,
738            &config.merge.remote,
739            config.coordination.as_ref(),
740        )
741        .await?;
742        let base_commit = fetched_base.commit.clone();
743
744        let roles = config.resolve_roles()?;
745        let max_parallel = config.graph.max_parallel.max(1);
746        let mut state = RunState::new(
747            repo.clone(),
748            base_branch,
749            base_commit.clone(),
750            String::new(),
751            config,
752        );
753        state.fetched_base = Some(fetched_base);
754        state.origin = Some(origin);
755
756        // Released before anything else touches the branch: a stale local
757        // branch is moved with `git branch -f`, which git refuses while an
758        // earlier attempt's worktree still has it checked out. Everything
759        // after this point that can fail puts the old run back.
760        // A hand-run review has no task, hence no earlier attempts, but a
761        // worktree of a dead run magi made may still be released.
762        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
763            earlier: Vec::new(),
764            home: crate::run::home(),
765            choice: None,
766            task_instruction: None,
767        });
768        let wait = repository_wait(&state, None);
769        let coordination_config = state.config.coordination.clone();
770        let coordination =
771            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
772                .await?;
773        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
774        if let Some(released) = &released {
775            state.event(
776                "release",
777                format!(
778                    "took `{branch}` over from run {}: its worktree was released: {}",
779                    crate::run::short_of(&released.old_id),
780                    released.audit
781                ),
782            );
783        }
784        // The owner's answer to an earlier divergence question is applied
785        // here: after the release (git will not move a checked-out branch)
786        // and before the sync that would otherwise ask again.
787        if let Some(choice) = takeover.choice.as_ref()
788            && let Err(e) =
789                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
790                    .await
791        {
792            if let Some(released) = &released {
793                released.restore(&repo, branch).await;
794            }
795            return Err(e.context("applying the owner's answer about the diverged branch"));
796        }
797        // An earlier attempt of this task that published the task head, and
798        // whose work is the branch under review, hands that head over: the
799        // review run updates the same pull request. Without one (a legacy
800        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
801        // own head and its own open pull request is adopted as before.
802        for id in takeover.earlier.iter().rev() {
803            if let Ok(old) = RunState::load_under(id, &takeover.home)
804                && let Some(head) = old.pr_head.clone()
805                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
806            {
807                state.pr_head = Some(head);
808                break;
809            }
810        }
811        drop(coordination);
812        let opened = Self::open_review(
813            &repo,
814            branch,
815            state,
816            roles,
817            max_parallel,
818            base_commit,
819            &takeover,
820        )
821        .await;
822        if opened.is_err()
823            && let Some(released) = &released
824        {
825            let _coordination = crate::repo_lock::SeatLease::operation_with(
826                &repo,
827                coordination_config.as_ref(),
828                &wait,
829            )
830            .await?;
831            released.restore(&repo, branch).await;
832        }
833        opened
834    }
835
836    /// Recover exactly the operator-selected public implementation, without touching old sources.
837    pub async fn recover_pr(
838        repo: &Path,
839        task: &str,
840        selection: crate::pr_recovery::Selection,
841        mut config: Config,
842    ) -> Result<Self> {
843        selection.configure(&mut config)?;
844        selection.check_task(task, &selection.instruction)?;
845        let base_branch =
846            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
847                .await?;
848        let fetched_base = resolve_base(
849            repo,
850            &base_branch,
851            &config.merge.remote,
852            config.coordination.as_ref(),
853        )
854        .await?;
855        let base_commit = fetched_base.commit.clone();
856        selection
857            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
858            .await?;
859        Box::pin(selection.verify_source(repo)).await?;
860        if let Some(proof) = &selection.source_proof {
861            anyhow::ensure!(
862                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
863                "source rebase base is outside the fetched landing base"
864            );
865        }
866        let roles = config.resolve_roles()?;
867        let max_parallel = config.graph.max_parallel.max(1);
868        let mut state = RunState::new(
869            repo.to_path_buf(),
870            base_branch,
871            base_commit.clone(),
872            selection.instruction.clone(),
873            config,
874        );
875        state.fetched_base = Some(fetched_base);
876        state.origin = Some(crate::run::Origin::queue(task));
877        state.pr_head = Some(selection.branch.clone());
878        state.pr_recovery = Some(selection.clone());
879        crate::pr_head::check_predecessors(&state, &selection.branch).await?;
880        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
881    }
882
883    async fn open_selected(
884        repo: &Path,
885        state: RunState,
886        roles: ResolvedRoles,
887        max_parallel: usize,
888        base_commit: String,
889        selection: crate::pr_recovery::Selection,
890    ) -> Result<Self> {
891        let mut state = state;
892        let branch = state.branch_for('A');
893        let wait = repository_wait(&state, None);
894        let coordination = coordinate_operation(&mut state, repo, &wait).await?;
895        git::git(repo, &["branch", &branch, selection.source()]).await?;
896        drop(coordination);
897        state.event(
898            "recovery",
899            format!(
900                "selected implementation {} for {} (public lease {})",
901                selection.source(),
902                selection.url,
903                selection.head
904            ),
905        );
906        let takeover = crate::handover::Takeover {
907            earlier: Vec::new(),
908            home: crate::run::home(),
909            choice: None,
910            task_instruction: Some(selection.instruction.clone()),
911        };
912        state.save()?;
913        let id = state.id.clone();
914        let opened = Self::open_review(
915            repo,
916            &branch,
917            state,
918            roles,
919            max_parallel,
920            base_commit,
921            &takeover,
922        )
923        .await;
924        match opened {
925            Ok(runner) => Ok(runner),
926            Err(e) => {
927                if let Ok(mut partial) = RunState::load(&id) {
928                    partial.status = RunStatus::Blocked;
929                    partial.driver_exited = true;
930                    partial.event(
931                        "recovery",
932                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
933                    );
934                    partial.save()?;
935                }
936                Err(crate::pr_recovery::StartupFailed {
937                    run: id,
938                    cause: format!("{e:#}"),
939                }
940                .into())
941            }
942        }
943    }
944
945    /// What the newest earlier attempt that reviewed `branch` leaves for this
946    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
947    /// no earlier attempts. Reads the earlier record only: ownership of its
948    /// worktree and branch is [`crate::handover`]'s business.
949    async fn inherit(
950        repo: &Path,
951        branch: &str,
952        base_commit: &str,
953        takeover: &crate::handover::Takeover,
954    ) -> Option<crate::recovery::RecoveryRecord> {
955        use crate::recovery::{Relation, assess};
956        let newest = takeover.earlier.last()?;
957        let mut chosen = None;
958        for id in takeover.earlier.iter().rev() {
959            if let Ok(old) = RunState::load_under(id, &takeover.home)
960                && !old.reviews.is_empty()
961                && (old.winner().is_some_and(|w| w.branch == branch)
962                    || old.pr_head.as_deref() == Some(branch))
963            {
964                chosen = Some(old);
965                break;
966            }
967        }
968        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
969            .await
970            .unwrap_or_default();
971        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
972            Some(last) if last.head == tip => Relation::Same,
973            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
974            _ => Relation::Diverged,
975        };
976        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
977        Some(assess(from, chosen.as_ref(), base_commit, relation))
978    }
979
980    /// The half of [`Runner::review_taking_over`] that can fail after an
981    /// earlier attempt's worktree was released.
982    async fn open_review(
983        repo: &Path,
984        branch: &str,
985        mut state: RunState,
986        roles: ResolvedRoles,
987        max_parallel: usize,
988        base_commit: String,
989        takeover: &crate::handover::Takeover,
990    ) -> Result<Self> {
991        let wait = repository_wait(&state, None);
992        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
993        if state.pr_recovery.is_none() {
994            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
995        }
996        // The commit subjects are the closest thing to a task statement that
997        // existing work carries, and the reviewers are told as much.
998        let start = review_base(
999            repo,
1000            &state.config.merge.remote,
1001            &state.base_branch,
1002            &base_commit,
1003            branch,
1004        )
1005        .await;
1006        let log = git::log_oneline(repo, &start, branch)
1007            .await
1008            .unwrap_or_default();
1009        let log_text = if log.trim().is_empty() {
1010            "(no commit messages)"
1011        } else {
1012            log.trim()
1013        };
1014        let mut instruction = format!(
1015            "Review the work already on branch `{branch}`. There is no task \
1016             statement: what the change claims to do is whatever its commits \
1017             say.\n\n{log_text}"
1018        );
1019        // A retry of a task whose earlier attempt left reviews behind carries
1020        // that attempt's evidence, and the task's own statement with it: the
1021        // completion criteria are the task's, not whatever the commits claim.
1022        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1023            if let Some(task) = takeover
1024                .task_instruction
1025                .as_deref()
1026                .filter(|t| !t.trim().is_empty())
1027                && rec.reviewed_head.is_some()
1028            {
1029                instruction = format!(
1030                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1031                    task.trim()
1032                );
1033                rec.criteria_inherited = true;
1034            }
1035            state.event("recovery", rec.summary());
1036            state.recovery = Some(rec);
1037        }
1038        if let Some(selected) = &state.pr_recovery {
1039            instruction = selected.instruction.clone();
1040        }
1041        state.task_instruction = takeover.task_instruction.clone();
1042        state.instruction = instruction;
1043        state.reviewed_commits = Some(
1044            git::subjects(repo, &start, branch)
1045                .await
1046                .unwrap_or_default(),
1047        );
1048
1049        // An attached worktree, so the fixer's commits land on the branch under
1050        // review rather than on a detached head nobody will look at again.
1051        let worktree = state.worktree_root().join("under-review");
1052        if let Some(parent) = worktree.parent() {
1053            tokio::fs::create_dir_all(parent).await.ok();
1054        }
1055        let path = worktree.to_string_lossy().to_string();
1056        git::git(repo, &["worktree", "add", &path, branch])
1057            .await
1058            .with_context(|| {
1059                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1060            })?;
1061
1062        let setup = match ensure_setup_config(&mut state, repo).await {
1063            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1064            Err(e) => Err(e),
1065        };
1066        if let Err(e) = setup {
1067            git::worktree_remove_clean(repo, &worktree).await.ok();
1068            return Err(e);
1069        }
1070        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1071            .await
1072            .unwrap_or(0);
1073        if commits == 0 {
1074            git::worktree_remove_clean(repo, &worktree).await.ok();
1075            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1076        }
1077        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1078            .await
1079            .map(|f| f.len())
1080            .unwrap_or(0);
1081        if files == 0
1082            && let (Ok(head_tree), Ok(base_tree)) = (
1083                git::tree_of(&worktree, "HEAD").await,
1084                git::tree_of(&worktree, &base_commit).await,
1085            )
1086            && head_tree == base_tree
1087        {
1088            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1089            git::worktree_remove_clean(repo, &worktree).await.ok();
1090            bail!(
1091                "`{branch}` at {} has a tree identical to base {}; this usually means \
1092                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1093                 against `{}/{branch}`) rather than an empty change",
1094                short(&head),
1095                short(&base_commit),
1096                state.config.merge.remote
1097            );
1098        }
1099        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1100            .await
1101            .unwrap_or_default();
1102
1103        state.candidates.push(Candidate {
1104            index: 0,
1105            label: 'A',
1106            // Not an agent id on purpose: nothing in the roster wrote this, and
1107            // the stats tables must not credit anyone with a win for it.
1108            agent: EXISTING_BRANCH.to_owned(),
1109            branch: branch.to_owned(),
1110            worktree,
1111            summary: String::new(),
1112            stat,
1113            files,
1114            commits,
1115            empty: false,
1116            failed: None,
1117            verified_noop: None,
1118            duration_ms: 0,
1119            folded: false,
1120        });
1121        state.tally = Some(Tally {
1122            first_choice: BTreeMap::from([('A', 0)]),
1123            borda: BTreeMap::new(),
1124            winner: 'A',
1125            rankings: 0,
1126            unanimous_initial: false,
1127            deliberated: false,
1128            changed_votes: 0,
1129            unanimous_final: false,
1130            tie_break: None,
1131            // No panel sat, so no quorum applies. Zero judges is the correct
1132            // number for work that never competed, and must not be reported as
1133            // a collapsed panel.
1134            judges: 0,
1135            present: 0,
1136            quorum: 0,
1137            met_quorum: true,
1138            uncontested: Some("review-only run: nothing competed".to_owned()),
1139        });
1140        state.status = RunStatus::Reviewing;
1141        state.prep_complete = Some(true);
1142        state.event(
1143            "start",
1144            format!(
1145                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1146                state.id
1147            ),
1148        );
1149        state.save()?;
1150        Ok(Self {
1151            state,
1152            roles,
1153            sem: Arc::new(Semaphore::new(max_parallel)),
1154            pause: Pause::new(),
1155            interrupt: Pause::new(),
1156        })
1157    }
1158
1159    /// Reopen an existing run.
1160    pub fn resume(id: &str) -> Result<Self> {
1161        let state = RunState::load(id)?;
1162        crate::pr_head::ensure_owner(&state)?;
1163        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1164            bail!(
1165                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1166            );
1167        }
1168        if let Some(to) = &state.released_to {
1169            bail!(
1170                "run {} cannot be resumed: its worktree was released to run {}",
1171                state.short(),
1172                crate::run::short_of(to)
1173            );
1174        }
1175        let roles = state.config.resolve_roles()?;
1176        if let Some(id) = &state.config.graph.selected_implementer
1177            && state.candidates.iter().any(|c| c.agent != *id)
1178        {
1179            bail!(
1180                "replacement implementer pin does not match the saved candidates; task must stay held"
1181            );
1182        }
1183        let max_parallel = state.config.graph.max_parallel.max(1);
1184        Ok(Self {
1185            state,
1186            roles,
1187            sem: Arc::new(Semaphore::new(max_parallel)),
1188            pause: Pause::new(),
1189            interrupt: Pause::new(),
1190        })
1191    }
1192
1193    /// Walk the graph to a terminal state, skipping nodes already recorded.
1194    ///
1195    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1196    /// the phone - ends here, so this is the one place a run that ended
1197    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1198    /// notification centre. Best-effort: see [`crate::notices::raise`].
1199    pub async fn execute(&mut self) -> Result<()> {
1200        if let Some(selection) = &self.state.pr_recovery
1201            && (selection.source_proof.is_some() || selection.source_run.is_some())
1202        {
1203            Box::pin(selection.verify_source(&self.state.repo)).await?;
1204        }
1205        // Queue claims cover daemon work; this also serializes manual drivers
1206        // consuming a repair answer, including the new review and decision.
1207        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1208            || self.state.land_approval.is_some()
1209            || self.state.merge_decision.is_some()
1210            || !self.state.merge_repairs.is_empty()
1211            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1212        {
1213            let claim = FixClaim::acquire(&self.state.dir())?;
1214            let disk = RunState::load(&self.state.id)?;
1215            let checkpoint = |s: &RunState| {
1216                (
1217                    s.merge_repairs.len(),
1218                    s.merge_repairs
1219                        .last()
1220                        .map(|r| (r.question.clone(), r.phase)),
1221                    s.land_approval
1222                        .as_ref()
1223                        .map(|a| (a.question.clone(), a.head.clone())),
1224                )
1225            };
1226            if checkpoint(&disk) != checkpoint(&self.state) {
1227                bail!(
1228                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1229                );
1230            }
1231            Some(claim)
1232        } else {
1233            None
1234        };
1235        let mut execution = self.claim_driver()?;
1236        let runner = &mut *execution.runner;
1237        let wait = runner.repository_wait();
1238        // Keep the graph's large future off the callers' stack, including
1239        // standalone task attempts running on a default-sized test thread.
1240        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1241        let result = match result {
1242            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1243                record_repository_block(&mut runner.state, &e);
1244                runner.park_here()?;
1245                Ok(())
1246            }
1247            other => other,
1248        };
1249        execution.driver.finish(&mut runner.state)?;
1250        let ended = if result.is_err() {
1251            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1252        } else {
1253            crate::notices::run_ended(&runner.state)
1254        };
1255        if let Some(notice) = ended {
1256            crate::notices::raise(notice);
1257        }
1258        result
1259    }
1260
1261    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1262        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1263    }
1264
1265    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1266        let wait = self.repository_wait();
1267        coordinate_operation(&mut self.state, cwd, &wait).await
1268    }
1269
1270    /// Validate ownership and persist the driver claim in one critical section.
1271    fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1272        let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1273            Ok(owner) => owner,
1274            Err(error) => {
1275                let detail = format!("worktree maintenance or ownership update: {error:#}");
1276                // Admission did not own run.json: retain this resource outcome
1277                // in memory for queue settlement without overwriting a concurrent
1278                // retirement or publication transaction.
1279                self.state.resource_block = Some(crate::run::ResourceBlock {
1280                    node: "start".into(),
1281                    kind: "worktree maintenance".into(),
1282                    detail: detail.clone(),
1283                    at: Timestamp::now(),
1284                });
1285                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1286            }
1287        };
1288        let disk = RunState::load(&self.state.id)?;
1289        crate::pr_head::ensure_owner(&disk)?;
1290        if disk.liveness(false) == crate::run::Liveness::Live {
1291            bail!("run already has a live driver; wait for its execution to end");
1292        }
1293        if let Some(to) = &disk.released_to {
1294            bail!(
1295                "run {} cannot continue: its worktree was released to run {}",
1296                self.state.short(),
1297                crate::run::short_of(to)
1298            );
1299        }
1300        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1301        let mut execution = DriverExecution {
1302            runner: self,
1303            driver,
1304        };
1305        // Declare after the execution guard: on any claim error this short
1306        // lock must drop before that guard reloads and releases ownership.
1307        let _owner = owner;
1308        let state = &mut execution.runner.state;
1309        // Moving again, so it is no longer parked. Set before the walk rather
1310        // than in `resume`, so every way of re-entering the graph clears it
1311        // and a card cannot claim a run is waiting to be resumed while the
1312        // agents are already working.
1313        state.parked = false;
1314        state.resource_wait = None;
1315        state.resource_block = None;
1316        // Any seat this state still lists as answering belongs to whatever
1317        // process last drove this run — this one included, if it crashed
1318        // mid-wave. Cleared and flushed immediately, before anything else
1319        // runs, so a resume can never show a seat as live when nothing is
1320        // asking it anything yet; the node that actually dispatches the next
1321        // wave repopulates it.
1322        state.clear_active();
1323        // Recorded in the same spot, and flushed together with the clear
1324        // above: this is the pid a reader checks (`RunState::liveness`) when
1325        // no daemon claim exists to answer "is a process still driving this
1326        // run" — a plain `magi run` / `magi review` typed into a terminal
1327        // claims nothing there. Always overwritten, never only-if-absent, so
1328        // a resumed run's stale pid from a previous, possibly-dead process
1329        // can never survive into this one's own report. Unlike
1330        // `clear_active`, this changes on every single `execute()` call, so
1331        // the save below is now unconditional rather than only-if-cleared.
1332        //
1333        // `driver_started_at` is recorded in the same breath, from this same
1334        // pid, so `liveness` can tell a live pid that is genuinely still us
1335        // apart from one the OS has since handed to an unrelated process —
1336        // see that field's own doc for why the pid alone is not enough.
1337        let pid = std::process::id();
1338        state.driver_pid = Some(pid);
1339        state.driver_started_at = crate::proc::process_started_at(pid);
1340        state.driver_exited = false;
1341        state.begin_pass();
1342        execution.driver.bind(state);
1343        state.save()?;
1344        execution.driver.record(state)?;
1345        Ok(execution)
1346    }
1347
1348    async fn execute_graph(&mut self) -> Result<()> {
1349        self.state.configure_recovery_resume()?;
1350        // Preservation may invalidate an old empty classification. Resolve it
1351        // before any resume shortcut can reuse a verdict or publish a branch.
1352        self.persist_candidates().await?;
1353        if self
1354            .state
1355            .merge_decision
1356            .as_ref()
1357            .is_some_and(|decision| !decision.closed)
1358            && self.state.status != RunStatus::Landing
1359        {
1360            if crate::merge_decision::resumable_hold(&self.state) {
1361                self.state.merge = Some(MergeOutcome {
1362                    mode: MergeMode::Pr,
1363                    ok: false,
1364                    detail: self
1365                        .state
1366                        .merge_decision
1367                        .as_ref()
1368                        .unwrap()
1369                        .target
1370                        .pr
1371                        .clone(),
1372                    empty: false,
1373                });
1374                self.run_land().await?;
1375                self.settle_questions();
1376                return Ok(());
1377            }
1378            bail!(
1379                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1380            );
1381        }
1382        if self.state.config.graph.selected_implementer.is_some()
1383            && self.state.status == RunStatus::Blocked
1384            && self.state.tally.is_none()
1385            && self.state.candidate_preservation.is_empty()
1386            && self.state.interrupted_implementations.is_empty()
1387        {
1388            bail!(
1389                "selected implementation is held; explicitly release the owning task to retry it"
1390            );
1391        }
1392
1393        if self.state.merge_repairs.last().is_some_and(|r| {
1394            matches!(
1395                r.phase,
1396                crate::run::MergeRepairPhase::Requested
1397                    | crate::run::MergeRepairPhase::Fixing
1398                    | crate::run::MergeRepairPhase::Reviewing
1399            )
1400        }) {
1401            if self.repair_merge_review().await? {
1402                self.run_land().await?;
1403            }
1404            self.settle_questions();
1405            return Ok(());
1406        }
1407        // Publication already passed review and gate. Resume only that node:
1408        // syncing to a newer base here could change the verified commit.
1409        if self.state.pr_publication_pending {
1410            self.merge().await?;
1411            if self.resource_halt()? {
1412                return Ok(());
1413            }
1414            self.state.save()?;
1415            self.settle_questions();
1416            return Ok(());
1417        }
1418        // A run that already lost its quorum never resumes into the verdict
1419        // machinery: `deliberate` and `vote` would otherwise clobber the
1420        // stalled marker back to Voting and the run would keep going past a
1421        // verdict that is no longer trustworthy. Everything already recorded is
1422        // kept, so the run stays resumable (or foldable) for a human to pick up.
1423        //
1424        // On --resume the run gets one chance to repair itself: the seats a
1425        // rate limit took out are re-asked. If their quota has since reset and
1426        // the quorum is restored, the run picks up and finishes; otherwise it
1427        // stays stale and still-resumable for a later retry. If it does not
1428        // recover, the returned status stays `Stalled` and nothing was
1429        // clobbered (the recovery only mutates entries for the lost seats).
1430        if self.state.status == RunStatus::Stalled {
1431            if self.recover_stall().await? {
1432                self.finish_after_tally().await?;
1433            } else {
1434                // Still below quorum: persist the marker and stay resumable.
1435                self.state.save()?;
1436            }
1437            return Ok(());
1438        }
1439        // A run parked inside `land` - watching CI, mid fix-round, or
1440        // waiting on the owner's merge approval - resumes directly into it,
1441        // never back through `prep`. Everything before `merge` already
1442        // concluded; that is the only way `status` reaches `Landing` in the
1443        // first place. Re-walking `review_loop` first would also be actively
1444        // wrong: its own status recomputation (see its doc) treats any
1445        // clean round as reason to set `status` to `Gating`, which would
1446        // clobber this marker before `merge` ever ran, and this run would
1447        // never find its way back into `land` at all.
1448        if self.state.status == RunStatus::Landing
1449            || crate::handover::landing_checkpoint(&self.state)
1450        {
1451            self.state.land_sync_pending = true;
1452            self.state.save()?;
1453            self.run_land().await?;
1454            // `run_land` may have settled the run right here - CI came back
1455            // green and the PR merged, say - without ever passing back
1456            // through `merge`'s own trailing call. Whatever it left `status`
1457            // as is what this has to read.
1458            self.settle_questions();
1459            return Ok(());
1460        }
1461        // A run parked at `merge` on the owner's word about a withheld PR text
1462        // resumes straight into `merge`. Re-walking the tail would sync to the
1463        // base again (possibly rebasing, possibly spending the fixer) after the
1464        // gate had passed, and then push a tree nobody reviewed or gated.
1465        if self.state.github_text.is_some()
1466            && self.state.merge.is_none()
1467            && self.state.status == RunStatus::Gating
1468        {
1469            self.merge().await?;
1470            if self.resource_halt()? {
1471                return Ok(());
1472            }
1473            self.state.save()?;
1474            return Ok(());
1475        }
1476        if let Err(e) = self.prep().await {
1477            if e.is::<crate::repo_lock::ResourceBlocked>() {
1478                return Err(e);
1479            }
1480            self.state.status = RunStatus::Blocked;
1481            self.state
1482                .prep_failure
1483                .get_or_insert_with(|| format!("{e:#}"));
1484            self.state
1485                .event("prep", format!("preparation stopped: {e:#}"));
1486            self.state.save()?;
1487            return Err(e);
1488        }
1489        if self.park_here()? {
1490            return Ok(());
1491        }
1492        self.advise().await?;
1493        if self.park_here()? {
1494            return Ok(());
1495        }
1496        self.implement().await?;
1497        if self.park_here()? {
1498            return Ok(());
1499        }
1500        // `after_implement` already saved the state and settled any open
1501        // questions when it set this; nothing later in the graph has
1502        // anything to judge.
1503        if self.state.status == RunStatus::VerifiedNoop {
1504            return Ok(());
1505        }
1506        self.judge().await?;
1507        if self.park_here()? {
1508            return Ok(());
1509        }
1510        self.deliberate().await?;
1511        if self.park_here()? {
1512            return Ok(());
1513        }
1514        self.vote().await?;
1515        if self.park_here()? {
1516            return Ok(());
1517        }
1518        self.tally()?;
1519        // A verdict that lost its quorum is not trustworthy: do not review,
1520        // gate, or merge on it. Everything already done is kept, so the run
1521        // stays resumable (or foldable); the human can replace the agent that
1522        // ran out of quota and pick it up.
1523        if self.state.status == RunStatus::Stalled {
1524            // Persist the stalled marker now — the normal end-of-execute save
1525            // below is below this early return, and without it a resumed run
1526            // would reload a pre-tally status and keep going.
1527            self.state.save()?;
1528            return Ok(());
1529        }
1530        self.finish_after_tally().await?;
1531        Ok(())
1532    }
1533
1534    /// A wave could not start its agents because a shared resource stayed busy
1535    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1536    /// resumable. Unstarted seats have no result; completed peers retain
1537    /// their saved candidate state.
1538    fn stop_for_resource(&mut self) -> Result<()> {
1539        // A wait abandoned because the run is parking stays a park.
1540        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1541        {
1542            self.state.status = RunStatus::Blocked;
1543        }
1544        self.state.save()
1545    }
1546
1547    /// Stop walking the graph if a node ended on a resource block.
1548    fn resource_halt(&mut self) -> Result<bool> {
1549        if self.state.resource_block.is_none() {
1550            return Ok(false);
1551        }
1552        self.stop_for_resource()?;
1553        Ok(true)
1554    }
1555
1556    /// Park here if asked to, recording it in the run's own timeline.
1557    ///
1558    /// Returns whether the caller should stop walking the graph. The state is
1559    /// saved either way by the node that just finished; this adds the event so
1560    /// the operator's card says why a run that is neither finished nor moving
1561    /// is sitting where it is.
1562    fn park_here(&mut self) -> Result<bool> {
1563        let parking = self.pause.parked() || self.interrupt.parked();
1564        if !parking && self.state.resource_block.is_some() {
1565            self.stop_for_resource()?;
1566            return Ok(true);
1567        }
1568        // Either handle asking is enough - see `Pause`'s own doc for why
1569        // they are never the same one. `interrupt` is checked second so a
1570        // reason it carries is preferred in the message below over a plain
1571        // shutdown park racing it at the same boundary.
1572        if !self.pause.parked() && !self.interrupt.parked() {
1573            return Ok(false);
1574        }
1575        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1576        let why = match cause.as_ref().map(ParkCause::label) {
1577            Some(reason) => format!(
1578                "parked after `{}` ({reason}) — resume to carry on from here",
1579                self.state.status.as_str()
1580            ),
1581            None => format!(
1582                "parked after `{}` — resume to carry on from here",
1583                self.state.status.as_str()
1584            ),
1585        };
1586        self.state.event("park", why);
1587        self.state
1588            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1589        self.state.parked = true;
1590        self.state.save()?;
1591        Ok(true)
1592    }
1593
1594    /// Only a new explicit operator approval may retry failed preparation.
1595    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1596        if self.state.prep_failure.is_none() {
1597            return Ok(());
1598        }
1599        let failed_at = self
1600            .state
1601            .events
1602            .iter()
1603            .rev()
1604            .find(|event| event.node == "prep")
1605            .map_or(self.state.updated_at, |event| event.at);
1606        anyhow::ensure!(
1607            approval_at > failed_at,
1608            "preparation retry requires a new explicit release after the failure"
1609        );
1610        self.state.prep_failure = None;
1611        self.state.prep_complete = Some(false);
1612        self.state
1613            .event("prep-retry", "operator approved retry of preparation");
1614        Ok(())
1615    }
1616
1617    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1618    pub fn on_pause(&mut self, pause: Pause) {
1619        self.pause = pause;
1620    }
1621
1622    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1623    /// scheduler asking this one run - and no other - to park so a task
1624    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1625    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1626    /// handle.
1627    pub fn watch_interrupt(&mut self, pause: Pause) {
1628        self.interrupt = pause;
1629    }
1630
1631    /// Abandon this run's own open questions, once `status` has actually
1632    /// settled rather than merely paused.
1633    ///
1634    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1635    /// either back up with the candidates, the review round and the seat
1636    /// sessions already on disk, so a question an implementer asked mid-round
1637    /// may still get a real answer read by a real resume. Only the statuses
1638    /// `resumable` excludes are actually final: the run merged, it reached
1639    /// `Ready` with nothing left to do, it failed outright with no
1640    /// established point to continue from, or every candidate agreed, with
1641    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1642    /// every one of those the seat that asked is gone for good, exactly like
1643    /// the run being deleted under `magi run rm` - so the same cleanup
1644    /// applies, worded for what actually happened instead of "the run was
1645    /// deleted".
1646    ///
1647    /// Best-effort and silent on success: called from every place `status`
1648    /// can land on one of those three, including ones a resumed run revisits,
1649    /// so it must cost nothing when there was nothing open to begin with.
1650    fn settle_questions(&mut self) {
1651        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1652            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1653        }
1654    }
1655
1656    /// The tail of the graph after a trustworthy tally: fold losers, review,
1657    /// gate, merge, and persist.
1658    async fn finish_after_tally(&mut self) -> Result<()> {
1659        // Before the losers are folded: their branches are what the graft
1660        // reads from.
1661        self.graft().await?;
1662        if self.resource_halt()? {
1663            return Ok(());
1664        }
1665        self.fold_losers().await?;
1666        // Before review starts, and again right before the gate: a run's
1667        // review rounds can themselves take long enough for the base to move
1668        // a second time, and the gate is the one node whose "green" gets
1669        // acted on.
1670        self.sync_to_base().await?;
1671        if self.resource_halt()? {
1672            return Ok(());
1673        }
1674        if self.state.status == RunStatus::AlreadyInBase {
1675            return Ok(());
1676        }
1677        self.review_loop().await?;
1678        if self.resource_halt()? {
1679            return Ok(());
1680        }
1681        self.sync_to_base().await?;
1682        if self.resource_halt()? {
1683            return Ok(());
1684        }
1685        if self.state.status == RunStatus::AlreadyInBase {
1686            return Ok(());
1687        }
1688        self.gate().await?;
1689        self.merge().await?;
1690        if self.resource_halt()? {
1691            return Ok(());
1692        }
1693        self.state.save()?;
1694        Ok(())
1695    }
1696
1697    // ---------------------------------------------------------------- prep
1698
1699    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1700    /// run with the step and its output in the event; it is never swallowed.
1701    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1702    /// resume starts over instead of finding candidates and missing seats.
1703    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1704        let result = match ensure_setup_config(&mut self.state, repo).await {
1705            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1706            Err(e) => Err(e),
1707        };
1708        if let Err(e) = result {
1709            git::worktree_remove_clean(repo, wt).await.ok();
1710            if in_prep {
1711                self.rollback_prep(repo).await;
1712            }
1713            self.state.status = RunStatus::Blocked;
1714            self.state.event(
1715                "setup",
1716                format!("worktree setup failed in {}: {e:#}", wt.display()),
1717            );
1718            self.state.save()?;
1719            return Err(e);
1720        }
1721        Ok(())
1722    }
1723
1724    /// Undo everything `prep` created, so a resume re-runs it whole.
1725    async fn rollback_prep(&mut self, repo: &Path) {
1726        let root = self.state.worktree_root();
1727        for c in std::mem::take(&mut self.state.candidates) {
1728            if matches!(
1729                git::worktree_remove_clean(repo, &c.worktree).await,
1730                Ok(true)
1731            ) {
1732                git::branch_delete(repo, &c.branch).await.ok();
1733            } else {
1734                self.state.candidates.push(c);
1735            }
1736        }
1737        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1738        for j in 1..=seats {
1739            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1740                .await
1741                .ok();
1742        }
1743        for k in 1..=self.state.config.graph.advisors {
1744            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1745                .await
1746                .ok();
1747        }
1748    }
1749
1750    async fn prep(&mut self) -> Result<()> {
1751        if let Some(reason) = &self.state.prep_failure {
1752            bail!(
1753                "candidate preparation previously failed: {reason}; correct the source before retrying"
1754            );
1755        }
1756        if self
1757            .state
1758            .prep_complete
1759            .unwrap_or(!self.state.candidates.is_empty())
1760        {
1761            return Ok(());
1762        }
1763        let repo = self.state.repo.clone();
1764        // Includes rollback, config, refs, checkout creation and setup. No
1765        // repository mutation may precede the real shared lock acquisition.
1766        let _coordination = self.coordinate(&repo).await?;
1767        self.rollback_prep(&repo).await;
1768        if !self.state.candidates.is_empty() {
1769            bail!(
1770                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1771            );
1772        }
1773        self.state.prep_complete = Some(false);
1774        self.state.status = RunStatus::Prep;
1775        self.state.save()?;
1776        let repo = self.state.repo.clone();
1777        let base = self.state.base_commit.clone();
1778        let plan = refs::plan(&repo, &self.state.seeds).await?;
1779        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1780        let root = self.state.worktree_root();
1781        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1782
1783        // The hook is the write-time half of the blindness contract; the
1784        // presentation filter in `blind` is the half that cannot be bypassed.
1785        let hooks_dir = self.state.dir().join("hooks");
1786        if self.state.config.blind.commit_msg_hook {
1787            std::fs::create_dir_all(&hooks_dir)
1788                .with_context(|| format!("create {}", hooks_dir.display()))?;
1789            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1790            let path = hooks_dir.join("commit-msg");
1791            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1792            make_executable(&path)?;
1793            // Ref-counted rather than a plain idempotent set: with more than
1794            // one run able to be in flight in the same repository at once
1795            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1796            // true?" check cannot tell "another run of mine still needs
1797            // this" from "nobody does", and the run that happens to finish
1798            // first would disable the hook out from under a sibling still
1799            // relying on it.
1800            git::acquire_worktree_config(&repo).await?;
1801            self.state.enabled_worktree_config = true;
1802        }
1803
1804        // `[worktree] setup` hides its products per worktree, which needs the
1805        // same per-worktree config the hook does. Held until fold, like it.
1806        ensure_setup_config(&mut self.state, &repo).await?;
1807
1808        for (index, (spec, label)) in self
1809            .roles
1810            .implementers
1811            .clone()
1812            .into_iter()
1813            .zip(labels)
1814            .enumerate()
1815        {
1816            let branch = self.state.branch_for(label);
1817            let worktree = root.join(format!("cand-{label}"));
1818            // Persist intent before Git can leave a partially created tree.
1819            // Candidate registration follows setup, so it cannot inventory
1820            // crashes or refusals during creation itself.
1821            self.state.event(
1822                "prep-intent",
1823                serde_json::json!({
1824                    "branch": branch,
1825                    "worktree": worktree,
1826                    "start": start,
1827                    "picks": plan.picks,
1828                })
1829                .to_string(),
1830            );
1831            self.state.save()?;
1832            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1833            if self.state.config.blind.commit_msg_hook {
1834                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1835            }
1836            git::local_exclude(&worktree, "/.magi/").await?;
1837            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1838                // Nothing of this prep may survive: a resume must start over,
1839                // not find half the candidates and skip the rest.
1840                git::worktree_remove_clean(&repo, &worktree).await.ok();
1841                git::branch_delete(&repo, &branch).await.ok();
1842                self.rollback_prep(&repo).await;
1843                self.state.status = RunStatus::Blocked;
1844                self.state
1845                    .event("prep", format!("worktree setup failed: {e:#}"));
1846                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1847                self.state.save()?;
1848                return Err(e);
1849            }
1850            self.state.candidates.push(Candidate {
1851                index,
1852                label,
1853                agent: spec.id.clone(),
1854                branch,
1855                worktree,
1856                summary: String::new(),
1857                stat: String::new(),
1858                files: 0,
1859                commits: 0,
1860                empty: false,
1861                failed: None,
1862                verified_noop: None,
1863                duration_ms: 0,
1864                folded: false,
1865            });
1866            self.state.save()?;
1867            for pick in &plan.picks {
1868                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1869                {
1870                    self.state.status = RunStatus::Blocked;
1871                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1872                    self.state
1873                        .event("prep", format!("cannot apply referenced commit: {e}"));
1874                    self.state.save()?;
1875                    return Err(e);
1876                }
1877            }
1878        }
1879
1880        for j in 1..=self.roles.judges.len() {
1881            let wt = root.join(format!("judge-{j}"));
1882            if !wt.exists() {
1883                git::worktree_add_detached(&repo, &wt, &base).await?;
1884                self.setup_seat_worktree(&repo, &wt, true).await?;
1885            }
1886        }
1887
1888        // Disposable, detached checkouts for the design-deliberation stage's
1889        // advisor seats — the same shape as the judges' above, at the same
1890        // base commit, since advisors also only ever read. Sized off the
1891        // configured count directly rather than a resolved roster: unlike
1892        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1893        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1894        // `prep` has no `ResolvedRoles` field to read a count from here.
1895        if self.state.config.graph.advise {
1896            for k in 1..=self.state.config.graph.advisors {
1897                let wt = root.join(format!("advisor-{k}"));
1898                if !wt.exists() {
1899                    git::worktree_add_detached(&repo, &wt, &base).await?;
1900                    self.setup_seat_worktree(&repo, &wt, true).await?;
1901                }
1902            }
1903        }
1904
1905        // A judge cannot tell it is looking at its own patch — the seats keep
1906        // separate conversations — but a panel that shares agents with the
1907        // field is less independent than it looks, and that is worth saying out
1908        // loud once per run rather than leaving it in the config.
1909        let authors: Vec<&str> = self
1910            .roles
1911            .implementers
1912            .iter()
1913            .map(|a| a.id.as_str())
1914            .collect();
1915        let overlap: Vec<String> = self
1916            .roles
1917            .judges
1918            .iter()
1919            .enumerate()
1920            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1921            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1922            .collect();
1923        if !overlap.is_empty() {
1924            let note = format!(
1925                "{} also authored a candidate; blind, but the panel is less \
1926                 independent than {} distinct agents would be",
1927                overlap.join(", "),
1928                self.roles.judges.len()
1929            );
1930            self.state.event("prep", note);
1931        }
1932
1933        self.state.event(
1934            "prep",
1935            format!(
1936                "{} candidates, {} judges, base {} ({})",
1937                self.state.candidates.len(),
1938                self.roles.judges.len(),
1939                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1940                self.state.base_branch
1941            ),
1942        );
1943        self.state.prep_complete = Some(true);
1944        self.state.status = RunStatus::Implementing;
1945        self.state.save()?;
1946        Ok(())
1947    }
1948
1949    // -------------------------------------------------------------- advise
1950
1951    /// The design-deliberation stage: independent, read-only advisor seats
1952    /// each sketch a design before any implementer touches the repository,
1953    /// and (when at least one produced a usable proposal) a synthesis seat
1954    /// blends them into a brief `implement` carries in every candidate's
1955    /// prompt.
1956    ///
1957    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1958    /// advisors` is the proposal count. Everything here is best-effort and
1959    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1960    /// that cannot reach quota, or a synthesis seat that produced nothing
1961    /// usable all leave `implement` exactly as it was before this stage
1962    /// existed — the task instruction alone — rather than failing the whole
1963    /// competition over an enrichment stage. Every outcome is still recorded
1964    /// as an event, so a run that got nothing from this stage says why.
1965    ///
1966    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1967    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1968    /// resumed run whose stage failed would re-run it, and re-spend the
1969    /// agent calls, on every reentry before `implement`.
1970    ///
1971    /// Also skipped once any candidate shows implementation progress — the
1972    /// exact predicate `implement` itself uses to decide a candidate is no
1973    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1974    /// is not enough: a run created by an older binary that predates this
1975    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1976    /// an already-`Implementing`-or-later run under this build would
1977    /// otherwise walk straight back through `prep` (a no-op once candidates
1978    /// exist) into this node and spawn every advisor seat against worktrees
1979    /// `prep` never recreated — after implementation has already started,
1980    /// which is exactly the invariant this stage exists to guarantee.
1981    async fn advise(&mut self) -> Result<()> {
1982        let implement_untouched = self
1983            .state
1984            .candidates
1985            .iter()
1986            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1987        if !self.state.config.graph.advise || self.state.advise_attempted {
1988            return Ok(());
1989        }
1990        if !implement_untouched {
1991            self.state.event(
1992                "advise",
1993                "skipping the design-deliberation stage: at least one \
1994                 candidate already shows implementation progress, so this \
1995                 run is past the point the stage exists to run before"
1996                    .to_owned(),
1997            );
1998            self.state.advise_attempted = true;
1999            self.state.save()?;
2000            return Ok(());
2001        }
2002        let run_id = self.state.id.clone();
2003        let prompts = self.state.config.prompts.clone();
2004        let instruction = self.state.instruction.clone();
2005        let language = self.state.config.graph.language.clone();
2006        let root = self.state.worktree_root();
2007        let n = self.state.config.graph.advisors;
2008        let where_recorded = self.state.dir().join("run.json");
2009
2010        let seats = match self.state.config.advisors() {
2011            Ok(seats) if !seats.is_empty() => seats,
2012            Ok(_) => {
2013                self.state.event(
2014                    "advise",
2015                    format!(
2016                        "[graph] advisors is 0; skipping the design-deliberation \
2017                         stage and continuing without a synthesis brief (see {})",
2018                        where_recorded.display()
2019                    ),
2020                );
2021                self.state.advise_attempted = true;
2022                self.state.save()?;
2023                return Ok(());
2024            }
2025            Err(e) => {
2026                self.state.event(
2027                    "advise",
2028                    format!(
2029                        "could not resolve advisor seats ({e:#}); continuing \
2030                         without a design-deliberation brief (see {})",
2031                        where_recorded.display()
2032                    ),
2033                );
2034                self.state.advise_attempted = true;
2035                self.state.save()?;
2036                return Ok(());
2037            }
2038        };
2039
2040        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2041        let artifacts = agent::artifacts_dir(&self.state.dir());
2042        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2043
2044        let mut jobs = Vec::new();
2045        for (i, spec) in seats.iter().cloned().enumerate() {
2046            let seat_key = format!("advisor-{}", i + 1);
2047            let seat = self.seat(&seat_key, &spec.id);
2048            jobs.push(SeatJob {
2049                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2050                spec,
2051                seat,
2052                cwd: worktrees[i % worktrees.len()].clone(),
2053                timeout,
2054                allow_write: false,
2055                sessions: false,
2056                artifacts: artifacts.clone(),
2057                stem: seat_key,
2058                handover: None,
2059            });
2060        }
2061
2062        self.state.event(
2063            "advise",
2064            format!(
2065                "{} advisor seat(s) sketching a design in parallel",
2066                jobs.len()
2067            ),
2068        );
2069        let mut quota_losses = Vec::new();
2070        let cache = self.state.config.cache_dir();
2071        let ctx = WaveCtx {
2072            carry_seats: false,
2073            pauses: Some((&self.pause, &self.interrupt)),
2074            run: &run_id,
2075            node: "advise",
2076            prompts: &prompts,
2077            cache: cache.as_deref(),
2078            round: None,
2079        };
2080        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2081        let results = ask_json_wave::<Proposal>(
2082            jobs,
2083            Arc::clone(&self.sem),
2084            self.state.config.graph.retries,
2085            &advisor_roster,
2086            &ctx,
2087            &mut quota_losses,
2088            &mut self.state,
2089            &|p: &Proposal| p.validate(),
2090        )
2091        .await;
2092        if self.state.resource_block.is_some() {
2093            return self.stop_for_resource();
2094        }
2095        self.state.quota.extend(quota_losses);
2096
2097        let mut records = Vec::with_capacity(results.len());
2098        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2099            let agent_id = seat.agent.clone();
2100            self.state.seats.insert(seat.key.clone(), seat);
2101            match res {
2102                Ok((proposal, out)) => {
2103                    self.state
2104                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2105                    records.push(advise::AdvisorRecord::proposed(
2106                        i + 1,
2107                        agent_id,
2108                        proposal,
2109                        out.duration_ms,
2110                    ));
2111                }
2112                Err(e) => {
2113                    self.state.event(
2114                        "advise",
2115                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2116                    );
2117                    records.push(advise::AdvisorRecord::failed(
2118                        i + 1,
2119                        agent_id,
2120                        e.to_string(),
2121                    ));
2122                }
2123            }
2124        }
2125
2126        let mut advice = advise::Advice {
2127            records,
2128            synthesis: None,
2129        };
2130        if advice.proposals().is_empty() {
2131            self.state.event(
2132                "advise",
2133                "no advisor produced a usable proposal; continuing without a \
2134                 synthesis brief"
2135                    .to_owned(),
2136            );
2137        } else {
2138            match self
2139                .synthesize_brief(
2140                    &advice,
2141                    &instruction,
2142                    &language,
2143                    &worktrees[0],
2144                    &artifacts,
2145                    &run_id,
2146                    &prompts,
2147                    cache.as_deref(),
2148                )
2149                .await
2150            {
2151                Ok(Some(text)) => {
2152                    self.state.event(
2153                        "advise",
2154                        "synthesized a design brief for the implementer".to_owned(),
2155                    );
2156                    advice.synthesis = Some(text);
2157                }
2158                Ok(None) => {
2159                    self.state.event(
2160                        "advise",
2161                        "the synthesis seat produced nothing usable; continuing \
2162                         without a design brief"
2163                            .to_owned(),
2164                    );
2165                }
2166                Err(e) => {
2167                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2168                        return Err(e);
2169                    }
2170                    self.state.event(
2171                        "advise",
2172                        format!("could not synthesize a design brief: {e:#}"),
2173                    );
2174                }
2175            }
2176        }
2177        advise::apply_reflection(&mut advice);
2178
2179        self.state.advice = Some(advice);
2180        self.state.advise_attempted = true;
2181        self.state.save()?;
2182        Ok(())
2183    }
2184
2185    /// The synthesis seat: reads every advisor's proposal and blends them
2186    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2187    /// out of [`Runner::advise`] only for readability — it is not called
2188    /// anywhere else.
2189    ///
2190    /// Picked the same way [`crate::talk`]'s standing conversation and
2191    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2192    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2193    /// order (a claude seat, else the first runnable agent in roster order)
2194    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2195    /// in [`crate::config`] for why a dedicated field exists here at all.
2196    #[allow(clippy::too_many_arguments)]
2197    async fn synthesize_brief(
2198        &mut self,
2199        advice: &advise::Advice,
2200        instruction: &str,
2201        language: &str,
2202        cwd: &Path,
2203        artifacts: &Path,
2204        run_id: &str,
2205        prompts: &Prompts,
2206        cache: Option<&Path>,
2207    ) -> Result<Option<String>> {
2208        let chain = agent::pick_chain(
2209            &self.state.config.agents,
2210            self.state.config.roles.synthesizer.as_ref(),
2211            &agent::installed,
2212            "synthesizer",
2213        )?;
2214        let proposals = advice.proposals();
2215        let mut prompt = prompt::with_overlay(
2216            prompt::synthesize_brief(instruction, &proposals, language),
2217            prompts.overlay("advise"),
2218        );
2219        if cache.is_some() {
2220            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2221            // below — see `prompt::build_cache_note`'s doc for why telling a
2222            // read-only seat to build through the shared cache is exactly how
2223            // a sandbox's write refusal gets misread as a defect.
2224            prompt.push('\n');
2225            prompt.push_str(&prompt::build_cache_note("advise", false));
2226        }
2227        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2228        // Each id is tried once, in order; a quota hit, error or unusable
2229        // answer moves to the next. The seat is single-turn (`sessions:
2230        // false`) and the prompt is the whole context, so a fallback agent
2231        // needs nothing carried over.
2232        let mut last = None;
2233        for (n, spec) in chain.iter().enumerate() {
2234            if n > 0 {
2235                self.state
2236                    .event("advise", format!("synthesis falling back to {}", spec.id));
2237            }
2238            let mut seat = self.seat("advise-synthesis", &spec.id);
2239            let wait = self.repository_wait();
2240            let outcome = wait
2241                .scope(agent::invoke(
2242                    spec,
2243                    &mut seat,
2244                    &Invocation {
2245                        scope: crate::agent::InvocationScope::Repository,
2246                        coordination: self.state.config.coordination.as_ref(),
2247                        fetched_base: None,
2248                        cwd,
2249                        prompt: &prompt,
2250                        timeout,
2251                        allow_write: false,
2252                        unsandboxed: false,
2253                        sessions: false,
2254                        artifacts,
2255                        stem: &if n == 0 {
2256                            "advise-synthesis".to_owned()
2257                        } else {
2258                            format!("advise-synthesis-{}", spec.id)
2259                        },
2260                        run: run_id,
2261                        node: "advise",
2262                        cache_dir: None,
2263                        attachments: &[],
2264                        writable: &[],
2265                    },
2266                ))
2267                .await;
2268            if outcome.is_ok() {
2269                self.state.seats.insert(seat.key.clone(), seat);
2270            }
2271            let advance = agent::chain_advances(&outcome);
2272            last = Some(outcome);
2273            if !advance {
2274                break;
2275            }
2276        }
2277        // Exhausted: the last attempt's result is what a single failed seat
2278        // would have produced.
2279        let out = last.expect("a chain holds at least one agent")?;
2280        if !out.usable() {
2281            return Ok(None);
2282        }
2283        let text =
2284            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2285        Ok((!text.trim().is_empty()).then_some(text))
2286    }
2287
2288    // ----------------------------------------------------------- implement
2289
2290    async fn implement(&mut self) -> Result<()> {
2291        self.persist_candidates().await?;
2292        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2293        // agent files with `magi task add` name the run that paid for it. The
2294        // prompt overlay is cloned alongside it because the waves borrow it
2295        // while `self` is mutably borrowed by the node's own bookkeeping.
2296        let run_id = self.state.id.clone();
2297        let prompts = self.state.config.prompts.clone();
2298        let todo: Vec<usize> = self
2299            .state
2300            .candidates
2301            .iter()
2302            .enumerate()
2303            .filter(|(_, c)| {
2304                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2305                    && c.failed.is_none()
2306                    && !c.empty
2307            })
2308            .map(|(i, _)| i)
2309            .collect();
2310        if todo.is_empty() {
2311            return self.after_implement();
2312        }
2313        self.state.status = RunStatus::Implementing;
2314
2315        let language = self.state.config.graph.language.clone();
2316        let github_language = self.state.config.graph.github_language.clone();
2317        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2318        let sessions = self.state.config.graph.sessions;
2319        let artifacts = agent::artifacts_dir(&self.state.dir());
2320        // The design-deliberation stage's blended brief, when `advise` found
2321        // one — carried into every implementer's prompt the same way
2322        // regardless of which candidate it is.
2323        let brief = self
2324            .state
2325            .advice
2326            .as_ref()
2327            .and_then(|a| a.synthesis.as_deref())
2328            .map(str::to_owned);
2329        let attachments = self.state.attachments.clone();
2330
2331        let mut jobs = Vec::new();
2332        for &i in &todo {
2333            let (index, label, worktree) = {
2334                let c = &self.state.candidates[i];
2335                (c.index, c.label, c.worktree.clone())
2336            };
2337            let spec = self.roles.implementers[index].clone();
2338            let seat_key = format!("impl-{label}");
2339            let seat = self.seat(&seat_key, &spec.id);
2340            let instruction = seeded_instruction(&self.state);
2341            jobs.push(SeatJob {
2342                spec,
2343                seat,
2344                prompt: prompt::implement_in(
2345                    &instruction,
2346                    &worktree.to_string_lossy(),
2347                    &language,
2348                    &github_language,
2349                    brief.as_deref(),
2350                    &attachments,
2351                ),
2352                cwd: worktree,
2353                timeout,
2354                allow_write: true,
2355                sessions,
2356                artifacts: artifacts.clone(),
2357                stem: format!("impl-{label}"),
2358                handover: None,
2359            });
2360        }
2361
2362        self.state.event(
2363            "implement",
2364            format!("{} candidates in parallel", jobs.len()),
2365        );
2366        // Kept so a seat whose CLI hung up can be asked again from the same
2367        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2368        // can update a seat's own entry once a fallback agent takes it over —
2369        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2370        // whichever agent actually answered, not the one that quota'd out.
2371        let mut sent = jobs.clone();
2372        let cache = self.state.config.cache_dir();
2373        let ctx = WaveCtx {
2374            carry_seats: false,
2375            pauses: Some((&self.pause, &self.interrupt)),
2376            run: &run_id,
2377            node: "implement",
2378            prompts: &prompts,
2379            cache: cache.as_deref(),
2380            round: None,
2381        };
2382        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2383        if !wave_blocked(&results) {
2384            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2385                .await;
2386            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2387                .await?;
2388            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2389                .await;
2390        }
2391        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2392
2393        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2394            && results
2395                .iter()
2396                .any(|(_, _, out)| FailClass::of(out).is_some());
2397        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2398            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2399                // Executed turns interrupted by disk pressure already carry
2400                // a preservation barrier from wave. Admission-only waits do
2401                // not. Keep the interrupted seat's session for recovery too.
2402                if self
2403                    .state
2404                    .candidate_preservation
2405                    .contains_key(&self.state.candidates[i].label)
2406                {
2407                    self.state.seats.insert(seat.key.clone(), seat);
2408                }
2409                continue;
2410            }
2411            let seat_key = seat.key.clone();
2412            // A quota fallback (`resume_seat_handovers`) may have handed this
2413            // seat to a different agent than the one `prep` recorded on the
2414            // candidate; the stats tables and any later fixer-defaults-to-
2415            // winner's-author lookup must credit whoever actually answered —
2416            // unless every fallback also quota'd out, in which case nobody
2417            // actually answered and crediting the last agent tried would
2418            // erase every earlier agent's own quota loss from the stats
2419            // tables instead of just this one seat's.
2420            let agent = seat.agent.clone();
2421            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2422            self.state.seats.insert(seat.key.clone(), seat);
2423            let label = self.state.candidates[i].label;
2424            self.state.interrupted_implementations.remove(&label);
2425
2426            let (summary, duration, failed, verified_claim) = match out {
2427                AgentOutcome::Ok(o) => {
2428                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2429                    let failed = (!o.usable()).then(|| {
2430                        if o.timed_out {
2431                            "agent timed out".to_owned()
2432                        } else {
2433                            format!(
2434                                "agent exited with {}",
2435                                crate::notices::exit_status(o.exit_code)
2436                            )
2437                        }
2438                    });
2439                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2440                    (text, o.duration_ms, failed, verified_claim)
2441                }
2442                // Left un-resumed by `resume_undelivered` (a dirty tree
2443                // already rescues the work, or there was no session left to
2444                // resume into) — reported like the ordinary failure it is,
2445                // never as if `o.text` (the CLI's raw error JSON) were an
2446                // answer.
2447                AgentOutcome::Dropped(o) => {
2448                    let why = o
2449                        .dropped
2450                        .as_ref()
2451                        .map(|d| d.why.as_str())
2452                        .unwrap_or("the CLI ended the stream without delivering its answer");
2453                    (
2454                        String::new(),
2455                        o.duration_ms,
2456                        Some(format!("the CLI dropped the stream ({why})")),
2457                        None,
2458                    )
2459                }
2460                AgentOutcome::Quota(o) => {
2461                    self.state.quota.push(QuotaLoss {
2462                        seat: seat_key,
2463                        node: "implement".to_owned(),
2464                        at: Timestamp::now(),
2465                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2466                    });
2467                    (
2468                        String::new(),
2469                        o.duration_ms,
2470                        Some("rate limited (quota); produced no change".to_owned()),
2471                        None,
2472                    )
2473                }
2474                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2475                    (String::new(), 0, Some(e), None)
2476                }
2477            };
2478
2479            // Save every completed turn before any fallible Git operation. A failed
2480            // rescue must not erase the reply or prevent collection of other seats.
2481            let c = &mut self.state.candidates[i];
2482            if !exhausted_the_fallback_chain {
2483                c.agent = agent;
2484            }
2485            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2486            c.duration_ms = duration;
2487            c.failed = failed;
2488            c.verified_noop = verified_claim;
2489            self.state
2490                .candidate_preservation
2491                .insert(label, "awaiting persistence".to_owned());
2492        }
2493        self.state.save()?;
2494        self.persist_candidates().await?;
2495
2496        if resource_blocked {
2497            return self.stop_for_resource();
2498        }
2499        if selected_failed {
2500            self.state.status = RunStatus::Blocked;
2501            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2502            self.state.save()?;
2503            bail!(
2504                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2505            );
2506        }
2507        self.after_implement()
2508    }
2509
2510    /// Retry only persistence after a completed implementation turn.
2511    async fn persist_candidates(&mut self) -> Result<()> {
2512        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2513        for label in labels {
2514            let i = self
2515                .state
2516                .candidates
2517                .iter()
2518                .position(|c| c.label == label)
2519                .context("pending candidate is missing")?;
2520            let c = self.state.candidates[i].clone();
2521            let interrupted = self.state.interrupted_implementations.contains(&label);
2522            let result = async {
2523                let wait = crate::repo_lock::WaitPolicy::new(
2524                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2525                    || false,
2526                );
2527                let _coordination =
2528                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2529                // Best effort evidence requires no index writes. A successful snapshot
2530                // never authorizes removal of a tree whose rescue failed.
2531                if let Err(e) = git::snapshot_uncommitted(
2532                    &c.worktree,
2533                    &self
2534                        .state
2535                        .dir()
2536                        .join("artifacts")
2537                        .join(format!("candidate-{label}-recovery")),
2538                )
2539                .await
2540                {
2541                    self.state.event(
2542                        "preserve",
2543                        format!("candidate {label} snapshot incomplete: {e:#}"),
2544                    );
2545                }
2546                let rescued = git::rescue_commit(
2547                    &c.worktree,
2548                    &format!("magi: candidate {label} (uncommitted work)"),
2549                )
2550                .await?;
2551                self.state.note_withheld("implement", &rescued.withheld);
2552                let base = &self.state.base_commit;
2553                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2554                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2555                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2556                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2557                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2558                if c.empty && commits > 0 && !patch.trim().is_empty()
2559                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2560                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2561                        || self.state.tally.is_some())
2562                {
2563                    // Keep the old classification and pending marker so every
2564                    // retry stops too. Retain decisions as recovery evidence;
2565                    // only a supported recovery operation may replace them.
2566                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2567                }
2568                let c = &mut self.state.candidates[i];
2569                c.commits = commits;
2570                c.stat = stat;
2571                c.files = files;
2572                let no_changes = commits == 0 || patch.trim().is_empty();
2573                c.empty = no_changes && !interrupted;
2574                if interrupted && no_changes {
2575                    c.failed = None;
2576                    c.verified_noop = None;
2577                }
2578                if !c.empty {
2579                    c.failed = None;
2580                    c.verified_noop = None;
2581                }
2582                self.state.event(
2583                    "preserve",
2584                    format!("candidate {label}: {files} files, {commits} commits"),
2585                );
2586                Ok::<_, anyhow::Error>(())
2587            }
2588            .await;
2589            match result {
2590                Ok(()) => {
2591                    self.state.candidate_preservation.remove(&label);
2592                    if !interrupted || self.state.candidates[i].files > 0 {
2593                        self.state.interrupted_implementations.remove(&label);
2594                    }
2595                }
2596                Err(e) => {
2597                    let reason = format!("candidate {label} preservation failed: {e:#}");
2598                    self.state.event("preserve", &reason);
2599                    self.state.candidate_preservation.insert(label, reason);
2600                }
2601            }
2602            if self.state.candidate_preservation.is_empty()
2603                && self.state.status == RunStatus::Blocked
2604                && self.state.candidates.iter().all(|c| c.failed.is_none())
2605            {
2606                // Persist the recovered phase with the cleared pending marker;
2607                // a crash between saves must not turn this into a selected-seat
2608                // failure hold on the next resume.
2609                self.state.status = RunStatus::Implementing;
2610                self.state.event(
2611                    "preserve",
2612                    "candidate persistence recovered; implementation can continue",
2613                );
2614            }
2615            self.state.save()?;
2616        }
2617        if !self.state.candidate_preservation.is_empty() {
2618            self.state.status = RunStatus::Blocked;
2619            self.state.save()?;
2620            bail!(
2621                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2622            );
2623        }
2624        Ok(())
2625    }
2626
2627    /// Ask again, once, for work a CLI did and then failed to hand over.
2628    ///
2629    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2630    /// status with an empty response and a usage report showing output tokens,
2631    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2632    /// seven minutes and 14,267 output tokens that arrived as an empty
2633    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2634    ///
2635    /// Two conditions, and both matter:
2636    ///
2637    /// - **Only when the tree is untouched.** Often the agent has already
2638    ///   written its files and only the closing message was lost; the rescue
2639    ///   commit below picks that up and there is nothing to ask for. Re-asking
2640    ///   then would pay for a second implementation of work already on disk.
2641    /// - **Once.** A CLI that drops one stream can drop the next, and this
2642    ///   node is the most expensive in the graph.
2643    ///
2644    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2645    /// dropped reply still carried its `conversation_id`, so the seat is asked
2646    /// to finish what it was doing rather than sent the whole task again. It
2647    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2648    /// node's - for the same reason a re-ranked judge does: restating finished
2649    /// work is not the work.
2650    ///
2651    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2652    /// same way until it resets, while an abandoned conversation is still
2653    /// there to be picked up.
2654    async fn resume_undelivered(
2655        &mut self,
2656        results: &mut [(usize, SeatState, AgentOutcome)],
2657        sent: &[SeatJob],
2658        prompts: &Prompts,
2659        run_id: &str,
2660    ) {
2661        for (wi, seat, out) in results.iter_mut() {
2662            let Some(dropped) = (match &*out {
2663                AgentOutcome::Dropped(o) => o.dropped.clone(),
2664                _ => None,
2665            }) else {
2666                continue;
2667            };
2668            let Some(job) = sent.get(*wi) else { continue };
2669            // Already on disk? Then only the closing message was lost.
2670            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2671                self.state.event(
2672                    "implement",
2673                    format!(
2674                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2675                         work is in the tree",
2676                        seat.key, dropped.output_tokens, dropped.why
2677                    ),
2678                );
2679                continue;
2680            }
2681            // The re-ask only makes sense as a resume: `resume_after_drop`
2682            // says nothing about the task, trusting the seat to still hold it.
2683            // Without a session to resume — sessions disabled, or this CLI's
2684            // drop shape happened not to carry a session id — that prompt
2685            // would open a brand-new conversation with no context at all,
2686            // which is worse than leaving this as the ordinary failure it
2687            // already is.
2688            if !has_context(&job.spec, seat, job.sessions) {
2689                self.state.event(
2690                    "implement",
2691                    format!(
2692                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2693                         is no session left to resume",
2694                        seat.key, dropped.output_tokens, dropped.why
2695                    ),
2696                );
2697                continue;
2698            }
2699            self.state.event(
2700                "implement",
2701                format!(
2702                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2703                     conversation",
2704                    seat.key, dropped.output_tokens, dropped.why
2705                ),
2706            );
2707            let mut retry = job.clone();
2708            retry.seat = seat.clone();
2709            retry.prompt = prompt::resume_after_drop(&dropped.why);
2710            retry.timeout = retry_budget(job.timeout, true);
2711            retry.stem = format!("{}-resume", job.stem);
2712            let cache = self.state.config.cache_dir();
2713            let ctx = WaveCtx {
2714                carry_seats: false,
2715                pauses: Some((&self.pause, &self.interrupt)),
2716                run: run_id,
2717                node: "implement",
2718                prompts,
2719                cache: cache.as_deref(),
2720                round: None,
2721            };
2722            let (resumed_seat, resumed) =
2723                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2724            *seat = resumed_seat;
2725            *out = resumed;
2726        }
2727    }
2728
2729    /// Fall an implement seat through to the next untried agent in the
2730    /// implementer roster when it lost to quota — or, since the handover was
2731    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2732    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2733    /// quota path itself being unchanged — instead of leaving the
2734    /// seat's loss final the moment one agent's account runs dry.
2735    ///
2736    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2737    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2738    /// truncates to the single slot rotation picked, so a solo task whose one
2739    /// implementer hits quota mid-run used to have nothing else to try. This
2740    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2741    /// unrotated roster — which is the only place the *other* candidates in
2742    /// the machine's roster still exist once `implementers` has been cut down
2743    /// to size.
2744    ///
2745    /// Walks forward from just past the seat's own original position in the
2746    /// roster, never wrapping back to the front: a later candidate slot (say
2747    /// `beta`, the roster's second entry) must fall through to the *next*
2748    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2749    /// almost certainly a different candidate's own agent already — and once
2750    /// the roster's tail is exhausted there is nothing left to fall through
2751    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2752    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2753    /// let this retry that id forever. The loop keeps falling through until
2754    /// an attempt lands something other than `Quota` or the roster's tail
2755    /// runs out of untried ids, at which point the seat is left exactly as
2756    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2757    /// `QuotaLoss` recorded, the candidate failed/empty.
2758    ///
2759    /// `sent` is taken mutably and updated with the fallback agent's spec:
2760    /// `resume_unconfirmed_commands`, which runs after this and also reads
2761    /// `sent`, must see whichever agent actually ended up answering the seat
2762    /// — reading the stale, original spec there would check session
2763    /// eligibility against the wrong CLI and could hand a fallback agent's
2764    /// session id to the agent that just lost the seat to quota.
2765    ///
2766    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2767    /// — `self.seat` only reuses state when the agent id is unchanged, so
2768    /// handing it a different id already gets this for free. Reusing the old
2769    /// seat would resume a different CLI's session as if it were a
2770    /// continuation of this one.
2771    ///
2772    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2773    /// a quota loss cuts an agent off mid-turn, so anything already in the
2774    /// tree is unfinished work, not a completed candidate a re-ask would pay
2775    /// for twice. A dirty tree is rescued into a commit first (the same
2776    /// neutral-identity rescue `implement`'s own outcome loop gives every
2777    /// candidate) so the next agent starts clean.
2778    ///
2779    /// The new agent gets the implementer's full prompt and full
2780    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2781    /// it has no session and no context, and is implementing the task from
2782    /// nothing, unlike a resumed drop which is only restating work already
2783    /// done.
2784    ///
2785    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2786    /// `implement` event, never into `self.state.quota` — that is what
2787    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2788    /// go unspent, and a seat that ultimately recovered on its second or
2789    /// third agent is not the stalled panel that check exists to catch. Only
2790    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2791    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2792    /// outcome loop already has — this helper never pushes to it itself.
2793    async fn resume_seat_handovers(
2794        &mut self,
2795        results: &mut [(usize, SeatState, AgentOutcome)],
2796        sent: &mut [SeatJob],
2797        prompts: &Prompts,
2798        run_id: &str,
2799    ) -> Result<()> {
2800        let instruction = seeded_instruction(&self.state);
2801        let language = self.state.config.graph.language.clone();
2802        let github_language = self.state.config.graph.github_language.clone();
2803        let brief = self
2804            .state
2805            .advice
2806            .as_ref()
2807            .and_then(|a| a.synthesis.as_deref())
2808            .map(str::to_owned);
2809        let attachments = self.state.attachments.clone();
2810        for (wi, seat, out) in results.iter_mut() {
2811            // Who holds the other candidate seats of this wave right now
2812            // (earlier handovers already written back to `sent`).
2813            let others: BTreeSet<String> = sent
2814                .iter()
2815                .enumerate()
2816                .filter(|(j, _)| j != wi)
2817                .map(|(_, j)| j.spec.id.clone())
2818                .collect();
2819            let Some(job) = sent.get_mut(*wi) else {
2820                continue;
2821            };
2822            // Where the seat's own original agent sits in the roster — the
2823            // fallback walk starts just past here, never at the front, so a
2824            // later candidate slot's quota loss does not fall back onto an
2825            // earlier slot's own agent.
2826            let start = self
2827                .roles
2828                .implementer_roster
2829                .iter()
2830                .position(|s| s.id == job.spec.id)
2831                .unwrap_or(0);
2832            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2833            let mut fallback_attempt = 0usize;
2834            let mut prev: Option<FailClass> = None;
2835            while let Some(cur) = FailClass::of(&*out) {
2836                if !should_hand_over(prev.as_ref(), &cur) {
2837                    break;
2838                }
2839                let Some(next) =
2840                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2841                        .cloned()
2842                else {
2843                    break;
2844                };
2845                tried.insert(next.id.clone());
2846                fallback_attempt += 1;
2847
2848                let coordination = self.coordinate(&job.cwd).await?;
2849                if let Ok(r) = git::rescue_commit(
2850                    &job.cwd,
2851                    &format!(
2852                        "magi: candidate {} (uncommitted work before {} fallback)",
2853                        seat.key,
2854                        if cur == FailClass::Quota {
2855                            "quota"
2856                        } else {
2857                            "handover"
2858                        }
2859                    ),
2860                )
2861                .await
2862                {
2863                    self.state.note_withheld("implement", &r.withheld);
2864                }
2865                drop(coordination);
2866
2867                record_handover(
2868                    &mut self.state,
2869                    "implement",
2870                    &seat.key,
2871                    &seat.agent,
2872                    &next.id,
2873                    &cur,
2874                    &fail_reason(&*out),
2875                );
2876                prev = Some(cur.clone());
2877
2878                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2879                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2880                // Kept in sync on `sent` itself, not just the local retry: a
2881                // later helper (`resume_unconfirmed_commands`) reads `sent`
2882                // after this one returns and must see whichever agent is now
2883                // occupying the seat, not the one that just quota'd out —
2884                // otherwise it would judge session/continuation eligibility
2885                // by the wrong CLI and could resend a fallback's session id
2886                // to the agent that lost it the seat in the first place.
2887                job.spec = next.clone();
2888                let mut retry = job.clone();
2889                retry.seat = new_seat;
2890                retry.prompt = prompt::implement_in(
2891                    &instruction,
2892                    &job.cwd.to_string_lossy(),
2893                    &language,
2894                    &github_language,
2895                    brief.as_deref(),
2896                    &attachments,
2897                );
2898                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2899                let cache = self.state.config.cache_dir();
2900                let ctx = WaveCtx {
2901                    carry_seats: false,
2902                    pauses: Some((&self.pause, &self.interrupt)),
2903                    run: run_id,
2904                    node: "implement",
2905                    prompts,
2906                    cache: cache.as_deref(),
2907                    round: None,
2908                };
2909                let (fallback_seat, fallback_out) = run_one(
2910                    retry,
2911                    Arc::clone(&self.sem),
2912                    &ctx,
2913                    &mut self.state,
2914                    fallback_attempt,
2915                )
2916                .await;
2917                *seat = fallback_seat;
2918                *out = fallback_out;
2919            }
2920        }
2921        Ok(())
2922    }
2923
2924    /// Ask an implement seat's own CLI to confirm what it started, once, when
2925    /// its reply reported a command whose completion status it never
2926    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2927    /// that does and does not mean.
2928    ///
2929    /// The completion contract this task asks for, extended to `implement`
2930    /// with the same signal `continue_fix_report` reads for the fixer,
2931    /// rather than a keyword search over the reply or a hard requirement on
2932    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2933    /// a candidate's CLI turn ended cleanly while a test run it had started
2934    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2935    /// named in it at all is untouched by this: `commands` is empty, so
2936    /// there is nothing to be unconfirmed.
2937    ///
2938    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2939    /// this is not about recovering edits that might already be on disk, it
2940    /// is about a result the seat itself never vouched for, which resuming
2941    /// asks for regardless of what the tree already holds. Bounded to one
2942    /// attempt for the same reason `resume_undelivered` is — this is the
2943    /// most expensive node in the graph — and a seat that still cannot
2944    /// confirm on that attempt is left as whatever its (possibly still
2945    /// unconfirmed) reply says; this does not invent a new "failed" reason
2946    /// for a candidate that otherwise produced a real, committed change.
2947    async fn resume_unconfirmed_commands(
2948        &mut self,
2949        results: &mut [(usize, SeatState, AgentOutcome)],
2950        sent: &[SeatJob],
2951        prompts: &Prompts,
2952        run_id: &str,
2953    ) {
2954        for (wi, seat, out) in results.iter_mut() {
2955            let AgentOutcome::Ok(o) = &*out else {
2956                continue;
2957            };
2958            if !has_unconfirmed_command(&o.commands) {
2959                continue;
2960            }
2961            let Some(job) = sent.get(*wi) else { continue };
2962            if !has_context(&job.spec, seat, job.sessions) {
2963                self.state.event(
2964                    "implement",
2965                    format!(
2966                        "{}: the reply named a command whose own CLI never confirmed the exit \
2967                         status of, but there is no session left to resume",
2968                        seat.key
2969                    ),
2970                );
2971                continue;
2972            }
2973            self.state.event(
2974                "implement",
2975                format!(
2976                    "{}: the reply named a command whose own CLI never confirmed the exit \
2977                     status of; resuming the conversation",
2978                    seat.key
2979                ),
2980            );
2981            let mut retry = job.clone();
2982            retry.seat = seat.clone();
2983            retry.prompt = prompt::resume_incomplete(
2984                "a command in your last reply had no confirmed exit status",
2985            );
2986            retry.timeout = retry_budget(job.timeout, true);
2987            retry.stem = format!("{}-confirm", job.stem);
2988            let cache = self.state.config.cache_dir();
2989            let ctx = WaveCtx {
2990                carry_seats: false,
2991                pauses: Some((&self.pause, &self.interrupt)),
2992                run: run_id,
2993                node: "implement",
2994                prompts,
2995                cache: cache.as_deref(),
2996                round: None,
2997            };
2998            let (resumed_seat, resumed) =
2999                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3000            *seat = resumed_seat;
3001            *out = resumed;
3002        }
3003    }
3004
3005    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3006    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3007    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3008    /// that motivated this.
3009    ///
3010    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3011    /// own nudge loop already covers for judge/review/vote seats, and not a
3012    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3013    /// implement seats: here the CLI turn genuinely finished while the node's
3014    /// own work — the fixer's account of what it did — had not. Gated purely
3015    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3016    /// reply, never on any wording in it, so a fixer whose valid, first-try
3017    /// `FixReport` happens to mention having waited on a background test is
3018    /// never resumed — the `Ok(report)` branch at the call site returns
3019    /// before this is ever invoked.
3020    ///
3021    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3022    /// attempt ([`retry_budget`]), nothing attempted once the session is
3023    /// gone, and a quota hit ends the loop immediately rather than retrying a
3024    /// rate limit that fails the same way again.
3025    async fn continue_fix_report(
3026        &mut self,
3027        mut seat: SeatState,
3028        parse_err: String,
3029        job: &SeatJob,
3030        prompts: &Prompts,
3031        run_id: &str,
3032        round: usize,
3033    ) -> (
3034        SeatState,
3035        Option<FixReport>,
3036        Option<String>,
3037        ContinuationRecord,
3038    ) {
3039        let mut last_err = parse_err;
3040        let mut cumulative_wait_ms = 0u64;
3041        let mut attempts = 0usize;
3042        loop {
3043            if !has_context(&job.spec, &seat, job.sessions) {
3044                self.state.event(
3045                    "fix",
3046                    format!(
3047                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3048                         session left to resume into"
3049                    ),
3050                );
3051                let outcome = if attempts == 0 {
3052                    ContinuationOutcome::NoSession
3053                } else {
3054                    ContinuationOutcome::Exhausted
3055                };
3056                return (
3057                    seat,
3058                    None,
3059                    Some(format!("unparsable fix report: {last_err}")),
3060                    ContinuationRecord {
3061                        attempts,
3062                        cumulative_wait_ms,
3063                        outcome,
3064                    },
3065                );
3066            }
3067            if attempts >= MAX_FIX_CONTINUATIONS {
3068                self.state.event(
3069                    "fix",
3070                    format!(
3071                        "round {round}: fixer's reply still had no adoption report after \
3072                         {attempts} continuation(s) ({last_err}); giving up"
3073                    ),
3074                );
3075                return (
3076                    seat,
3077                    None,
3078                    Some(format!(
3079                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3080                    )),
3081                    ContinuationRecord {
3082                        attempts,
3083                        cumulative_wait_ms,
3084                        outcome: ContinuationOutcome::Exhausted,
3085                    },
3086                );
3087            }
3088            attempts += 1;
3089            self.state.event(
3090                "fix",
3091                format!(
3092                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3093                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3094                ),
3095            );
3096            let mut retry = job.clone();
3097            retry.seat = seat.clone();
3098            retry.prompt = prompt::resume_incomplete(&last_err);
3099            retry.timeout = retry_budget(job.timeout, true);
3100            retry.stem = format!("{}-continue{attempts}", job.stem);
3101            let cache = self.state.config.cache_dir();
3102            let ctx = WaveCtx {
3103                carry_seats: false,
3104                pauses: Some((&self.pause, &self.interrupt)),
3105                run: run_id,
3106                node: "fix",
3107                prompts,
3108                cache: cache.as_deref(),
3109                round: Some(round),
3110            };
3111            let (resumed_seat, resumed_out) = run_one(
3112                retry,
3113                Arc::clone(&self.sem),
3114                &ctx,
3115                &mut self.state,
3116                attempts,
3117            )
3118            .await;
3119            seat = resumed_seat;
3120            match resumed_out {
3121                AgentOutcome::Ok(o) => {
3122                    cumulative_wait_ms += o.duration_ms;
3123                    match verdict::extract_json::<FixReport>(&o.text) {
3124                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3125                            self.state.event(
3126                                "fix",
3127                                format!(
3128                                    "round {round}: fixer's adoption report recovered after \
3129                                     {attempts} continuation(s)"
3130                                ),
3131                            );
3132                            return (
3133                                seat,
3134                                Some(report),
3135                                None,
3136                                ContinuationRecord {
3137                                    attempts,
3138                                    cumulative_wait_ms,
3139                                    outcome: ContinuationOutcome::Resumed,
3140                                },
3141                            );
3142                        }
3143                        // The report parsed, but this same reply's own
3144                        // CommandEvidence — the identical record `state.jobs`
3145                        // renders — names a command whose CLI never
3146                        // confirmed an exit status. Read together, that is
3147                        // not a resolved answer: keep nudging rather than
3148                        // accept a report standing next to a command the
3149                        // seat's own CLI cannot vouch for.
3150                        Ok(_) => {
3151                            last_err = "the reply parsed, but it reported a command whose own CLI \
3152                                 never confirmed an exit status"
3153                                .to_owned();
3154                        }
3155                        Err(e) => last_err = e.to_string(),
3156                    }
3157                }
3158                AgentOutcome::Quota(o) => {
3159                    cumulative_wait_ms += o.duration_ms;
3160                    self.state.quota.push(QuotaLoss {
3161                        seat: seat.key.clone(),
3162                        node: "fix".to_owned(),
3163                        at: Timestamp::now(),
3164                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3165                    });
3166                    self.state.event(
3167                        "fix",
3168                        format!(
3169                            "round {round}: continuation rate limited (quota); not retrying now"
3170                        ),
3171                    );
3172                    return (
3173                        seat,
3174                        None,
3175                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3176                        ContinuationRecord {
3177                            attempts,
3178                            cumulative_wait_ms,
3179                            outcome: ContinuationOutcome::QuotaLost,
3180                        },
3181                    );
3182                }
3183                AgentOutcome::Dropped(o) => {
3184                    cumulative_wait_ms += o.duration_ms;
3185                    let why = o
3186                        .dropped
3187                        .as_ref()
3188                        .map(|d| d.why.as_str())
3189                        .unwrap_or("the CLI ended the stream without delivering its answer");
3190                    last_err = format!("the CLI dropped the stream ({why})");
3191                }
3192                AgentOutcome::Failed(e) => last_err = e,
3193                // The continuation was never started; the report stays lost
3194                // and the caller records that, without any handover.
3195                AgentOutcome::ResourceBlocked(e) => {
3196                    return (
3197                        seat,
3198                        None,
3199                        Some(format!("the continuation was not started: {e}")),
3200                        ContinuationRecord {
3201                            attempts: attempts.saturating_sub(1),
3202                            cumulative_wait_ms,
3203                            outcome: ContinuationOutcome::NoSession,
3204                        },
3205                    );
3206                }
3207            }
3208        }
3209    }
3210
3211    fn after_implement(&mut self) -> Result<()> {
3212        if !self.state.candidate_preservation.is_empty() {
3213            bail!("candidate preservation incomplete; judging is unsafe");
3214        }
3215        // Scan every candidate patch once the set is complete.
3216        if self.state.leaks.is_empty() {
3217            let cfg = self.state.config.blind.clone();
3218            let mut leaks = Vec::new();
3219            for c in &self.state.candidates {
3220                let Some(patch) =
3221                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3222                else {
3223                    continue;
3224                };
3225                leaks.extend(blind::scan(
3226                    &format!("candidate {} patch", c.label),
3227                    &patch,
3228                    &cfg.vendor_tokens,
3229                ));
3230            }
3231            if !leaks.is_empty() {
3232                let summary = leaks
3233                    .iter()
3234                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3235                    .collect::<Vec<_>>()
3236                    .join(", ");
3237                match cfg.on_leak {
3238                    LeakPolicy::Fail => {
3239                        self.state.status = RunStatus::Failed;
3240                        self.state
3241                            .event("blind", format!("vendor text in a patch: {summary}"));
3242                        self.state.leaks = leaks;
3243                        self.state.save()?;
3244                        self.settle_questions();
3245                        bail!(
3246                            "blind.on_leak = \"fail\" and vendor text reached a \
3247                             judged patch: {summary}"
3248                        );
3249                    }
3250                    LeakPolicy::Redact => self.state.event(
3251                        "blind",
3252                        format!("redacting vendor text for judging: {summary}"),
3253                    ),
3254                    LeakPolicy::Warn => self.state.event(
3255                        "blind",
3256                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3257                    ),
3258                }
3259                self.state.leaks = leaks;
3260            }
3261        }
3262
3263        if self.state.viable().is_empty() {
3264            if self.state.all_candidates_verified_noop() {
3265                // Every candidate agreed, with evidence the adoption guard
3266                // accepted, that nothing belongs in this worktree. That is
3267                // not the same fact as a candidate that simply failed to
3268                // write anything, and settling it as an ordinary `Failed`
3269                // (see `SCHEMA`'s doc for schema 10) is what let two of
3270                // task 391f's attempts burn a retry each re-discovering the
3271                // same already-landed fix. Terminal either way, so `judge`
3272                // must never run over an empty candidate set — unlike the
3273                // `Failed` branch below this returns `Ok`, not an error:
3274                // nothing here failed.
3275                self.state.status = RunStatus::VerifiedNoop;
3276                self.state.save()?;
3277                self.settle_questions();
3278                return Ok(());
3279            }
3280            self.state.status = RunStatus::Failed;
3281            self.state.save()?;
3282            self.settle_questions();
3283            bail!("no candidate produced a change; nothing to judge");
3284        }
3285        self.state.status = RunStatus::Judging;
3286        self.state.save()?;
3287        Ok(())
3288    }
3289
3290    // --------------------------------------------------------------- judge
3291
3292    async fn judge(&mut self) -> Result<()> {
3293        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3294        // agent files with `magi task add` name the run that paid for it. The
3295        // prompt overlay is cloned alongside it because the waves borrow it
3296        // while `self` is mutably borrowed by the node's own bookkeeping.
3297        let run_id = self.state.id.clone();
3298        let prompts = self.state.config.prompts.clone();
3299        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3300            return Ok(());
3301        }
3302        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3303        if viable.len() == 1 {
3304            // Recorded so this is a one-time event: `judgements` stays empty
3305            // either way, which without this flag is indistinguishable from
3306            // "not yet judged" on the next reentry — and status is left
3307            // untouched, so a later node's conclusion (e.g. `Blocked` after
3308            // the review budget ran out) survives a resume instead of being
3309            // clobbered back to `Judging` by this node running again.
3310            self.state.judge_skipped = true;
3311            self.state.event(
3312                "judge",
3313                format!(
3314                    "only candidate {} produced a change; judging skipped",
3315                    viable[0].label
3316                ),
3317            );
3318            self.state.save()?;
3319            return Ok(());
3320        }
3321        self.state.status = RunStatus::Judging;
3322
3323        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3324        let language = self.state.config.graph.language.clone();
3325        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3326        let sessions = self.state.config.graph.sessions;
3327        let artifacts = agent::artifacts_dir(&self.state.dir());
3328        let root = self.state.worktree_root();
3329        let base_short = short(&self.state.base_commit);
3330
3331        let mut jobs = Vec::new();
3332        let mut orders = Vec::new();
3333        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3334            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3335            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3336            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3337            let seat_key = format!("judge-{}", j + 1);
3338            let seat = self.seat(&seat_key, &spec.id);
3339            jobs.push(SeatJob {
3340                prompt: prompt::judge(
3341                    &self.state.instruction,
3342                    &views,
3343                    self.roles.judges.len(),
3344                    &base_short,
3345                    &language,
3346                ),
3347                spec,
3348                seat,
3349                cwd: root.join(format!("judge-{}", j + 1)),
3350                timeout,
3351                allow_write: false,
3352                sessions,
3353                artifacts: artifacts.clone(),
3354                stem: format!("judge-{}", j + 1),
3355                handover: None,
3356            });
3357        }
3358
3359        self.state.event(
3360            "judge",
3361            format!(
3362                "{} judges ranking {} candidates blind",
3363                jobs.len(),
3364                viable.len()
3365            ),
3366        );
3367        let labels_for_check = labels.clone();
3368        let mut quota_losses = Vec::new();
3369        let cache = self.state.config.cache_dir();
3370        let ctx = WaveCtx {
3371            carry_seats: false,
3372            pauses: Some((&self.pause, &self.interrupt)),
3373            run: &run_id,
3374            node: "judge",
3375            prompts: &prompts,
3376            cache: cache.as_deref(),
3377            round: None,
3378        };
3379        let results = ask_json_wave::<Ranking>(
3380            jobs,
3381            Arc::clone(&self.sem),
3382            self.state.config.graph.retries,
3383            &self.roles.judge_roster,
3384            &ctx,
3385            &mut quota_losses,
3386            &mut self.state,
3387            &move |r: &Ranking| r.validate(&labels_for_check),
3388        )
3389        .await;
3390        if self.state.resource_block.is_some() {
3391            return self.stop_for_resource();
3392        }
3393        self.state.quota.extend(quota_losses);
3394
3395        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3396            let agent_id = seat.agent.clone();
3397            self.state.seats.insert(seat.key.clone(), seat);
3398            let mut record = Judgement {
3399                judge: j + 1,
3400                seat: format!("judge-{}", j + 1),
3401                agent: agent_id,
3402                ranking: Vec::new(),
3403                reasons: BTreeMap::new(),
3404                confidence: None,
3405                strengths: BTreeMap::new(),
3406                order: orders[j].clone(),
3407                failed: None,
3408                duration_ms: 0,
3409            };
3410            match res {
3411                Ok((ranking, out)) => {
3412                    record.ranking = ranking.normalized();
3413                    record.reasons = ranking.reasons;
3414                    record.strengths = ranking.strengths;
3415                    record.confidence = ranking.confidence;
3416                    record.duration_ms = out.duration_ms;
3417                    self.state.event(
3418                        "judge",
3419                        format!(
3420                            "judge {} ranked {}",
3421                            j + 1,
3422                            record.ranking.iter().collect::<String>()
3423                        ),
3424                    );
3425                }
3426                Err(e) => {
3427                    record.failed = Some(e.to_string());
3428                    self.state
3429                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3430                }
3431            }
3432            self.state.judgements.push(record);
3433            self.state.save()?;
3434        }
3435        Ok(())
3436    }
3437
3438    // ---------------------------------------------------------- deliberate
3439
3440    async fn deliberate(&mut self) -> Result<()> {
3441        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3442        // agent files with `magi task add` name the run that paid for it. The
3443        // prompt overlay is cloned alongside it because the waves borrow it
3444        // while `self` is mutably borrowed by the node's own bookkeeping.
3445        let run_id = self.state.id.clone();
3446        let prompts = self.state.config.prompts.clone();
3447        if !self.state.deliberation.is_empty() {
3448            return Ok(());
3449        }
3450        let tops: Vec<char> = self
3451            .state
3452            .judgements
3453            .iter()
3454            .filter_map(|j| j.ranking.first().copied())
3455            .collect();
3456        let rounds = self.state.config.graph.deliberate_rounds;
3457        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3458            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3459                self.state.event(
3460                    "deliberate",
3461                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3462                );
3463            }
3464            self.state.status = RunStatus::Voting;
3465            self.state.save()?;
3466            return Ok(());
3467        }
3468
3469        self.state.status = RunStatus::Deliberating;
3470        self.state.event(
3471            "deliberate",
3472            format!(
3473                "split: first choices were {} — opening {rounds} round(s)",
3474                tops.iter().collect::<String>()
3475            ),
3476        );
3477
3478        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3479        let language = self.state.config.graph.language.clone();
3480        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3481        let sessions = self.state.config.graph.sessions;
3482        let artifacts = agent::artifacts_dir(&self.state.dir());
3483        let root = self.state.worktree_root();
3484        let base_short = short(&self.state.base_commit);
3485
3486        // Judges argue in sequence so that a turn can answer the one before it;
3487        // that is the difference between deliberation and three parallel
3488        // monologues.
3489        for round in 1..=rounds {
3490            let mut turns: Vec<DeliberationTurn> = Vec::new();
3491            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3492                if self.state.judgements[j].failed.is_some() {
3493                    continue;
3494                }
3495                let seat_key = format!("judge-{}", j + 1);
3496                let spec = self.occupant(&seat_key, spec);
3497                let mut seat = self.seat(&seat_key, &spec.id);
3498                let transcript = self.transcript(&turns, j);
3499                let build = |context: Option<&str>| {
3500                    prompt::deliberate(
3501                        &self.state.instruction,
3502                        context,
3503                        &transcript,
3504                        round,
3505                        rounds,
3506                        &language,
3507                    )
3508                };
3509                let block = self.candidate_block(&viable, &base_short);
3510                let full = build(Some(&block));
3511                let text = if has_context(&spec, &seat, sessions) {
3512                    build(None)
3513                } else {
3514                    full.clone()
3515                };
3516                let job = SeatJob {
3517                    spec,
3518                    seat: seat.clone(),
3519                    prompt: text,
3520                    cwd: root.join(format!("judge-{}", j + 1)),
3521                    timeout,
3522                    allow_write: false,
3523                    sessions,
3524                    artifacts: artifacts.clone(),
3525                    stem: format!("delib-{round}-judge-{}", j + 1),
3526                    handover: Some(full),
3527                };
3528                let cache = self.state.config.cache_dir();
3529                let ctx = WaveCtx {
3530                    carry_seats: false,
3531                    pauses: Some((&self.pause, &self.interrupt)),
3532                    run: &run_id,
3533                    node: "deliberate",
3534                    prompts: &prompts,
3535                    cache: cache.as_deref(),
3536                    round: None,
3537                };
3538                // A turn is never nudged (`retries` 0): a failed seat is
3539                // handed to the next roster agent, which gets the full
3540                // context. An empty answer is a turn, not a failure.
3541                let mut losses = Vec::new();
3542                let mut results = ask_wave_with::<String>(
3543                    vec![job],
3544                    Arc::clone(&self.sem),
3545                    0,
3546                    &self.roles.judge_roster,
3547                    &ctx,
3548                    &mut losses,
3549                    &mut self.state,
3550                    &|text: &str| {
3551                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3552                    },
3553                )
3554                .await;
3555                self.state.quota.extend(losses);
3556                let (updated, res, _) = results.pop().expect("one job in, one result out");
3557                seat = updated;
3558                let agent_id = seat.agent.clone();
3559                self.state.seats.insert(seat.key.clone(), seat);
3560                let body = match res {
3561                    Ok((body, _)) => body,
3562                    // Skip the seat; a CLI's raw error JSON is never read as
3563                    // this judge's position.
3564                    Err(e) => {
3565                        self.state
3566                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3567                        continue;
3568                    }
3569                };
3570                let tentative = verdict::extract_json::<Position>(&body)
3571                    .ok()
3572                    .and_then(|p| p.tentative)
3573                    .and_then(|s| s.trim().chars().next())
3574                    .map(|c| c.to_ascii_uppercase());
3575                self.state.event(
3576                    "deliberate",
3577                    format!(
3578                        "round {round}: judge {} now favours {}",
3579                        j + 1,
3580                        tentative.map_or("—".to_owned(), |c| c.to_string())
3581                    ),
3582                );
3583                turns.push(DeliberationTurn {
3584                    judge: j + 1,
3585                    agent: agent_id,
3586                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3587                    tentative,
3588                });
3589            }
3590            self.state
3591                .deliberation
3592                .push(DeliberationRound { round, turns });
3593            self.state.save()?;
3594        }
3595
3596        self.state.status = RunStatus::Voting;
3597        self.state.save()?;
3598        Ok(())
3599    }
3600
3601    // ---------------------------------------------------------------- vote
3602
3603    async fn vote(&mut self) -> Result<()> {
3604        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3605        // agent files with `magi task add` name the run that paid for it. The
3606        // prompt overlay is cloned alongside it because the waves borrow it
3607        // while `self` is mutably borrowed by the node's own bookkeeping.
3608        let run_id = self.state.id.clone();
3609        let prompts = self.state.config.prompts.clone();
3610        if !self.state.votes.is_empty() {
3611            return Ok(());
3612        }
3613        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3614        if viable.len() == 1 {
3615            return Ok(());
3616        }
3617        self.state.status = RunStatus::Voting;
3618
3619        let language = self.state.config.graph.language.clone();
3620        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3621        let sessions = self.state.config.graph.sessions;
3622        let artifacts = agent::artifacts_dir(&self.state.dir());
3623        let root = self.state.worktree_root();
3624        let base_short = short(&self.state.base_commit);
3625        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3626
3627        let mut jobs = Vec::new();
3628        let mut seats_at = Vec::new();
3629        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3630            if self
3631                .state
3632                .judgements
3633                .get(j)
3634                .is_some_and(|r| r.failed.is_some())
3635            {
3636                continue;
3637            }
3638            let seat_key = format!("judge-{}", j + 1);
3639            let spec = self.occupant(&seat_key, spec);
3640            let seat = self.seat(&seat_key, &spec.id);
3641            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3642            let text = if has_context(&spec, &seat, sessions) {
3643                prompt::final_vote(&viable, &language)
3644            } else {
3645                full.clone()
3646            };
3647            jobs.push(SeatJob {
3648                spec,
3649                seat,
3650                prompt: text,
3651                cwd: root.join(format!("judge-{}", j + 1)),
3652                timeout,
3653                allow_write: false,
3654                sessions,
3655                artifacts: artifacts.clone(),
3656                stem: format!("vote-judge-{}", j + 1),
3657                handover: Some(full),
3658            });
3659            seats_at.push(j);
3660        }
3661
3662        self.state.event(
3663            "vote",
3664            format!(
3665                "collecting {} final votes one by one, privately",
3666                jobs.len()
3667            ),
3668        );
3669        let allowed = viable.clone();
3670        let mut quota_losses = Vec::new();
3671        let cache = self.state.config.cache_dir();
3672        let ctx = WaveCtx {
3673            carry_seats: false,
3674            pauses: Some((&self.pause, &self.interrupt)),
3675            run: &run_id,
3676            node: "vote",
3677            prompts: &prompts,
3678            cache: cache.as_deref(),
3679            round: None,
3680        };
3681        let results = ask_json_wave::<FinalVote>(
3682            jobs,
3683            Arc::clone(&self.sem),
3684            self.state.config.graph.retries,
3685            &self.roles.judge_roster,
3686            &ctx,
3687            &mut quota_losses,
3688            &mut self.state,
3689            &move |v: &FinalVote| match v.label() {
3690                Some(c) if allowed.contains(&c) => Ok(()),
3691                other => bail!("vote {other:?} is not one of {allowed:?}"),
3692            },
3693        )
3694        .await;
3695        if self.state.resource_block.is_some() {
3696            return self.stop_for_resource();
3697        }
3698        self.state.quota.extend(quota_losses);
3699
3700        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3701            let agent_id = seat.agent.clone();
3702            self.state.seats.insert(seat.key.clone(), seat);
3703            let initial = self
3704                .state
3705                .judgements
3706                .get(j)
3707                .and_then(|r| r.ranking.first().copied());
3708            let mut record = VoteRecord {
3709                judge: j + 1,
3710                agent: agent_id,
3711                vote: None,
3712                reason: String::new(),
3713                changed: false,
3714            };
3715            match res {
3716                Ok((v, _)) => {
3717                    record.vote = v.label();
3718                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3719                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3720                    self.state.event(
3721                        "vote",
3722                        format!(
3723                            "judge {} voted {}{}",
3724                            j + 1,
3725                            record.vote.unwrap_or('?'),
3726                            if record.changed { " (changed)" } else { "" }
3727                        ),
3728                    );
3729                }
3730                Err(e) => {
3731                    self.state
3732                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3733                }
3734            }
3735            self.state.votes.push(record);
3736            self.state.save()?;
3737        }
3738        Ok(())
3739    }
3740
3741    // --------------------------------------------------------------- tally
3742
3743    fn tally(&mut self) -> Result<()> {
3744        if self.state.tally.is_some() {
3745            return Ok(());
3746        }
3747        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3748        let tops: Vec<char> = self
3749            .state
3750            .judgements
3751            .iter()
3752            .filter_map(|j| j.ranking.first().copied())
3753            .collect();
3754        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3755
3756        // A judge whose private vote failed still counted once, in the initial
3757        // ranking; using it beats discarding a whole seat.
3758        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3759        let mut cast: Vec<char> = Vec::new();
3760        for (i, j) in self.state.judgements.iter().enumerate() {
3761            let vote = self
3762                .state
3763                .votes
3764                .iter()
3765                .find(|v| v.judge == i + 1)
3766                .and_then(|v| v.vote)
3767                .or_else(|| j.ranking.first().copied());
3768            if let Some(v) = vote {
3769                *first_choice.entry(v).or_insert(0) += 1;
3770                cast.push(v);
3771            }
3772        }
3773
3774        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3775        for j in &self.state.judgements {
3776            let n = j.ranking.len();
3777            for (pos, label) in j.ranking.iter().enumerate() {
3778                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3779            }
3780        }
3781
3782        let best = first_choice.values().copied().max().unwrap_or(0);
3783        let mut leaders: Vec<char> = first_choice
3784            .iter()
3785            .filter(|(_, v)| **v == best)
3786            .map(|(k, _)| *k)
3787            .collect();
3788        let mut tie_break = None;
3789        if leaders.len() > 1 {
3790            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3791            let borda_leaders: Vec<char> = leaders
3792                .iter()
3793                .copied()
3794                .filter(|l| borda[l] == top_borda)
3795                .collect();
3796            tie_break = Some(if borda_leaders.len() == 1 {
3797                format!(
3798                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3799                    leaders.len()
3800                )
3801            } else {
3802                format!(
3803                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3804                    leaders.len()
3805                )
3806            });
3807            leaders = borda_leaders;
3808            leaders.sort_unstable();
3809        }
3810        let winner = *leaders
3811            .first()
3812            .or(viable.first())
3813            .context("no candidate to declare a winner from")?;
3814
3815        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3816        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3817        let deliberated = !self.state.deliberation.is_empty();
3818
3819        // Whose verdict is this? A rate-limited seat is absent even if it
3820        // ranked before the limit hit, so presence is measured against the
3821        // recorded losses, not just "did a ranking ever appear".
3822        let quota_seats: std::collections::BTreeSet<&str> =
3823            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3824        let mut present = 0usize;
3825        for (i, j) in self.state.judgements.iter().enumerate() {
3826            if quota_seats.contains(j.seat.as_str()) {
3827                continue;
3828            }
3829            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3830            let voted = self
3831                .state
3832                .votes
3833                .iter()
3834                .any(|v| v.judge == i + 1 && v.vote.is_some());
3835            if ranked || voted {
3836                present += 1;
3837            }
3838        }
3839        // Strict majority of the configured panel. A bare majority is real
3840        // signal we can act on, while a minority verdict must never stand in
3841        // for a healthy one. A one-candidate run needs no panel at all, and
3842        // `judges` stays `0` rather than the roster size a panel that never
3843        // sat would otherwise be credited with.
3844        let needs_quorum = viable.len() > 1;
3845        let judges_total = if needs_quorum {
3846            self.roles.judges.len()
3847        } else {
3848            0
3849        };
3850        let quorum = if needs_quorum {
3851            judges_total / 2 + 1
3852        } else {
3853            0
3854        };
3855        let met_quorum = !needs_quorum || present >= quorum;
3856        let uncontested = (!needs_quorum).then(|| {
3857            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3858        });
3859
3860        self.state.event(
3861            "tally",
3862            match &uncontested {
3863                Some(reason) => format!("winner {winner} — {reason}"),
3864                None => format!(
3865                    "winner {winner} — votes {} | initial {} | {} changed | \
3866                     {present}/{judges_total} judges{}",
3867                    first_choice
3868                        .iter()
3869                        .map(|(k, v)| format!("{k}:{v}"))
3870                        .collect::<Vec<_>>()
3871                        .join(" "),
3872                    if unanimous_initial {
3873                        "unanimous"
3874                    } else {
3875                        "split"
3876                    },
3877                    changed_votes,
3878                    if met_quorum {
3879                        String::new()
3880                    } else {
3881                        format!(" — below quorum ({quorum} required)")
3882                    },
3883                ),
3884            },
3885        );
3886        if !met_quorum {
3887            self.state.event(
3888                "stall",
3889                format!(
3890                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3891                     the run stops here, resumable"
3892                ),
3893            );
3894        }
3895        self.state.tally = Some(Tally {
3896            first_choice,
3897            borda,
3898            winner,
3899            rankings: tops.len(),
3900            unanimous_initial,
3901            deliberated,
3902            changed_votes,
3903            unanimous_final,
3904            tie_break,
3905            judges: judges_total,
3906            present,
3907            quorum,
3908            met_quorum,
3909            uncontested,
3910        });
3911        self.state.status = if met_quorum {
3912            RunStatus::Reviewing
3913        } else {
3914            RunStatus::Stalled
3915        };
3916        self.state.save()?;
3917        Ok(())
3918    }
3919
3920    // ------------------------------------------------------------- recover
3921
3922    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3923    /// resumed toward completion once the transient cause clears.
3924    ///
3925    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3926    /// it toward the quorum, which is exactly the set of seats whose absence
3927    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3928    /// not depend on which node happened to hit the limit), or an ordinary
3929    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3930    /// seat is never disturbed.
3931    ///
3932    /// A seat that now answers with a usable ranking is "recovered": its
3933    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3934    /// `tally` counts it present again), and its vote re-collected. A seat that
3935    /// still fails keeps its loss and stays absent.
3936    ///
3937    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3938    /// to review/gate/merge), `false` when it is still below quorum (the run
3939    /// stays `Stalled`, still resumable for a later retry).
3940    #[allow(clippy::too_many_lines)]
3941    async fn recover_stall(&mut self) -> Result<bool> {
3942        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3943        // agent files with `magi task add` name the run that paid for it. The
3944        // prompt overlay is cloned alongside it because the waves borrow it
3945        // while `self` is mutably borrowed by the node's own bookkeeping.
3946        let run_id = self.state.id.clone();
3947        let prompts = self.state.config.prompts.clone();
3948        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3949        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3950        // is what keeps a non-quota collapse — or a quota loss recorded at the
3951        // deliberate node — from being a permanent dead-end on `--resume`.
3952        let quota_seats: BTreeSet<&str> =
3953            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3954        let absent: Vec<String> = self
3955            .state
3956            .judgements
3957            .iter()
3958            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3959            .map(|j| j.seat.clone())
3960            .collect();
3961        if absent.is_empty() {
3962            return Ok(false);
3963        }
3964        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3965        if viable.len() <= 1 {
3966            return Ok(false);
3967        }
3968        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3969        let language = self.state.config.graph.language.clone();
3970        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3971        let sessions = self.state.config.graph.sessions;
3972        let artifacts = agent::artifacts_dir(&self.state.dir());
3973        let root = self.state.worktree_root();
3974        let base_short = short(&self.state.base_commit);
3975        let candidates: Vec<Candidate> = viable.clone();
3976
3977        // Map each absent seat key to its 0-based position in `roles.judges`.
3978        let mut positions: Vec<usize> = absent
3979            .iter()
3980            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3981            .collect();
3982        if positions.is_empty() {
3983            return Ok(false);
3984        }
3985        positions.sort_unstable();
3986        positions.dedup();
3987
3988        // Re-rank the lost seats, one blind prompt each.
3989        let mut judge_jobs = Vec::new();
3990        for &j in &positions {
3991            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3992            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3993            let seat_key = format!("judge-{}", j + 1);
3994            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3995            let seat = self.seat(&seat_key, &spec.id);
3996            judge_jobs.push(SeatJob {
3997                spec,
3998                seat,
3999                prompt: prompt::judge(
4000                    &self.state.instruction,
4001                    &views,
4002                    self.roles.judges.len(),
4003                    &base_short,
4004                    &language,
4005                ),
4006                cwd: root.join(seat_key),
4007                timeout,
4008                allow_write: false,
4009                sessions,
4010                artifacts: artifacts.clone(),
4011                stem: format!("judge-{}-recover", j + 1),
4012                handover: None,
4013            });
4014        }
4015
4016        let labels_for_check = labels.clone();
4017        let mut judge_losses = Vec::new();
4018        let retries = self.state.config.graph.retries;
4019        let cache = self.state.config.cache_dir();
4020        let ctx = WaveCtx {
4021            carry_seats: false,
4022            pauses: Some((&self.pause, &self.interrupt)),
4023            run: &run_id,
4024            node: "judge",
4025            prompts: &prompts,
4026            cache: cache.as_deref(),
4027            round: None,
4028        };
4029        let results = ask_json_wave::<Ranking>(
4030            judge_jobs,
4031            Arc::clone(&self.sem),
4032            retries,
4033            &self.roles.judge_roster,
4034            &ctx,
4035            &mut judge_losses,
4036            &mut self.state,
4037            &move |r: &Ranking| r.validate(&labels_for_check),
4038        )
4039        .await;
4040        if self.state.resource_block.is_some() {
4041            self.stop_for_resource()?;
4042            return Ok(false);
4043        }
4044
4045        // Refresh the judgement of every seat that ranked again.
4046        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4047        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4048            let agent_id = seat.agent.clone();
4049            self.state.seats.insert(seat.key.clone(), seat);
4050            let record = &mut self.state.judgements[j];
4051            match res {
4052                Ok((ranking, out)) => {
4053                    record.agent = agent_id;
4054                    record.ranking = ranking.normalized();
4055                    record.reasons = ranking.reasons;
4056                    record.strengths = ranking.strengths;
4057                    record.confidence = ranking.confidence;
4058                    record.failed = None;
4059                    record.duration_ms = out.duration_ms;
4060                    recovered.insert(j);
4061                    self.state.event(
4062                        "recover",
4063                        format!("judge {} ranked again after the limit", j + 1),
4064                    );
4065                }
4066                Err(e) => {
4067                    self.state
4068                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4069                }
4070            }
4071        }
4072
4073        // Re-ask the votes of the seats that recovered a ranking.
4074        let mut vote_jobs = Vec::new();
4075        let mut vote_pos: Vec<usize> = Vec::new();
4076        for &j in &recovered {
4077            let seat_key = format!("judge-{}", j + 1);
4078            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4079            let seat = self.seat(&seat_key, &spec.id);
4080            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4081            let text = if has_context(&spec, &seat, sessions) {
4082                prompt::final_vote(&labels, &language)
4083            } else {
4084                full.clone()
4085            };
4086            vote_jobs.push(SeatJob {
4087                spec,
4088                seat,
4089                prompt: text,
4090                cwd: root.join(seat_key),
4091                timeout,
4092                allow_write: false,
4093                sessions,
4094                artifacts: artifacts.clone(),
4095                stem: format!("vote-judge-{}-recover", j + 1),
4096                handover: Some(full),
4097            });
4098            vote_pos.push(j);
4099        }
4100        let allowed = labels.clone();
4101        let mut vote_losses = Vec::new();
4102        let vote_retries = self.state.config.graph.retries;
4103        let vote_cache = self.state.config.cache_dir();
4104        let ctx = WaveCtx {
4105            carry_seats: false,
4106            pauses: Some((&self.pause, &self.interrupt)),
4107            run: &run_id,
4108            node: "vote",
4109            prompts: &prompts,
4110            cache: vote_cache.as_deref(),
4111            round: None,
4112        };
4113        let votes = ask_json_wave::<FinalVote>(
4114            vote_jobs,
4115            Arc::clone(&self.sem),
4116            vote_retries,
4117            &self.roles.judge_roster,
4118            &ctx,
4119            &mut vote_losses,
4120            &mut self.state,
4121            &move |v: &FinalVote| match v.label() {
4122                Some(c) if allowed.contains(&c) => Ok(()),
4123                other => bail!("vote {other:?} is not one of {allowed:?}"),
4124            },
4125        )
4126        .await;
4127        if self.state.resource_block.is_some() {
4128            self.stop_for_resource()?;
4129            return Ok(false);
4130        }
4131        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4132            let agent_id = seat.agent.clone();
4133            self.state.seats.insert(seat.key.clone(), seat);
4134            match res {
4135                Ok((v, _)) => {
4136                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4137                        rec.vote = v.label();
4138                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4139                    } else {
4140                        self.state.votes.push(VoteRecord {
4141                            judge: j + 1,
4142                            agent: agent_id,
4143                            vote: v.label(),
4144                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4145                            changed: false,
4146                        });
4147                    }
4148                    self.state.event(
4149                        "recover",
4150                        format!("judge {} voted again after the limit", j + 1),
4151                    );
4152                }
4153                Err(e) => {
4154                    self.state
4155                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4156                }
4157            }
4158        }
4159
4160        // A seat that ranked again is present even if its re-vote failed —
4161        // `tally` falls back to the initial ranking's first choice — so clear
4162        // its quota loss. Seats that still fail keep theirs and stay absent.
4163        let recovered_keys: BTreeSet<String> = recovered
4164            .iter()
4165            .map(|&j| format!("judge-{}", j + 1))
4166            .collect();
4167        self.state
4168            .quota
4169            .retain(|q| !recovered_keys.contains(&q.seat));
4170        // A seat that hit the limit again is a fresh loss, not the old one:
4171        // replace the stale entry so the history stays one-per-seat and the
4172        // daemon can tell this attempt's loss from a previous session's.
4173        for loss in judge_losses.into_iter().chain(vote_losses) {
4174            if recovered_keys.contains(&loss.seat) {
4175                continue;
4176            }
4177            self.state.quota.retain(|q| q.seat != loss.seat);
4178            self.state.quota.push(loss);
4179        }
4180
4181        // Recompute the verdict from the refreshed panel.
4182        self.state.tally = None;
4183        self.tally()?;
4184        Ok(self
4185            .state
4186            .tally
4187            .as_ref()
4188            .map(|t| t.met_quorum)
4189            .unwrap_or(false))
4190    }
4191
4192    // ---------------------------------------------------------------- graft
4193
4194    /// Port the strengths the judges attributed to the losing candidates onto
4195    /// the winner, once, between the tally and the review loop.
4196    ///
4197    /// Best-effort by construction: it returns nothing and fails nothing. A
4198    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4199    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4200    /// the run carries on as if the phase were off. See [`graft`] for the
4201    /// decisions and `AGENTS.md` for the invariants.
4202    ///
4203    /// The one exception is a rollback that could not be confirmed: the
4204    /// rejected commit may still be the winner's tip, so the run is blocked
4205    /// (resumable; the record stays `Running`, which makes a resume retry the
4206    /// restore) instead of reviewing, gating and merging it.
4207    async fn graft(&mut self) -> Result<()> {
4208        let Err(e) = self.graft_inner().await else {
4209            return Ok(());
4210        };
4211        if e.is::<crate::repo_lock::ResourceBlocked>() {
4212            return Err(e);
4213        }
4214        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4215            self.state.status = RunStatus::Blocked;
4216            self.state.event("graft", format!("blocked: {e:#}"));
4217            self.state.save()?;
4218            return Err(e);
4219        }
4220        // An error after the `Running` record was saved may have left the
4221        // fixer's work in the tree: the record must not stay unsettled
4222        // (a later resume would reset past the review fixes), so confirm the
4223        // restore now, or block.
4224        if let Some(rec) = self.state.graft.clone()
4225            && !rec.outcome.settled()
4226        {
4227            if let Some(winner) = self.state.winner().cloned() {
4228                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4229                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4230                        return Err(re);
4231                    }
4232                    self.state.status = RunStatus::Blocked;
4233                    self.state.event("graft", format!("blocked: {re:#}"));
4234                    self.state.save()?;
4235                    return Err(re);
4236                }
4237            }
4238            if let Some(g) = self.state.graft.as_mut() {
4239                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4240            }
4241        }
4242        self.state.event("graft", format!("skipped: {e:#}"));
4243        if let Err(e) = self.state.save() {
4244            tracing::warn!("save after graft: {e:#}");
4245        }
4246        Ok(())
4247    }
4248
4249    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4250    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4251    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4252        let _coordination = self.coordinate(wt).await?;
4253        crate::recovery::restore_repair(wt, pre)
4254            .await
4255            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4256    }
4257
4258    /// A skip is a run event, recorded once however often the node is re-entered.
4259    fn graft_skipped(&mut self, why: &str) {
4260        let message = format!("skipped: {why}");
4261        if !self
4262            .state
4263            .events
4264            .iter()
4265            .any(|e| e.node == "graft" && e.message == message)
4266        {
4267            self.state.event("graft", message);
4268        }
4269    }
4270
4271    async fn graft_inner(&mut self) -> Result<()> {
4272        // A round that was started and never finished (crash, kill): the
4273        // fixer may have left anything in the tree, so put the winner back.
4274        if let Some(rec) = self.state.graft.clone()
4275            && !rec.outcome.settled()
4276        {
4277            // Reviews exist only if the graft ended without its record being
4278            // settled (a swallowed error): the tree has moved on since, and
4279            // a reset to `pre` would discard the review fixes.
4280            if self.state.reviews.is_empty()
4281                && let Some(winner) = self.state.winner().cloned()
4282            {
4283                self.graft_restore(&winner.worktree, &rec.pre).await?;
4284            }
4285            if rec.outcome == GraftOutcome::WaitingForRestore {
4286                self.state.graft = None;
4287                self.state.save()?;
4288            } else {
4289                if let Some(g) = self.state.graft.as_mut() {
4290                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4291                }
4292                self.state.event(
4293                    "graft",
4294                    "rolled back: the previous round was interrupted; the winner is untouched",
4295                );
4296                self.state.save()?;
4297                return Ok(());
4298            }
4299        }
4300        let plan = match graft::plan(&self.state) {
4301            Ok(plan) => plan,
4302            Err(skip) => {
4303                if skip != graft::Skip::AlreadyDone {
4304                    self.graft_skipped(skip.reason());
4305                    self.state.save()?;
4306                }
4307                return Ok(());
4308            }
4309        };
4310        let Some(winner) = self.state.winner().cloned() else {
4311            return Ok(());
4312        };
4313        let wt = winner.worktree.clone();
4314        if !git::is_clean(&wt).await? {
4315            self.graft_skipped("the winner's worktree has uncommitted changes");
4316            self.state.save()?;
4317            return Ok(());
4318        }
4319        let pre = git::rev_parse(&wt, "HEAD").await?;
4320        // Saved before the fixer is called: a stop from here on is repaired
4321        // by the branch above on the next entry.
4322        self.state.graft = Some(GraftRecord {
4323            pre: pre.clone(),
4324            post: None,
4325            lines: None,
4326            limit: None,
4327            agent: None,
4328            outcome: GraftOutcome::Running,
4329            requested: graft::requested(&plan),
4330            adoption: None,
4331            adoption_error: None,
4332        });
4333        self.state.event(
4334            "graft",
4335            format!(
4336                "porting strengths of candidate(s) {} onto {}",
4337                plan.strengths
4338                    .iter()
4339                    .map(|(l, _, _)| l.to_string())
4340                    .collect::<Vec<_>>()
4341                    .join(", "),
4342                plan.winner
4343            ),
4344        );
4345        self.state.save()?;
4346
4347        let fix_prompt = prompt::graft(
4348            &self.state.instruction,
4349            plan.winner,
4350            &plan.strengths,
4351            &self.state.config.graph.language,
4352        );
4353        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4354        let sessions = self.state.config.graph.sessions;
4355        let artifacts = agent::artifacts_dir(&self.state.dir());
4356        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4357        let (_, seat, out) = self
4358            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4359                SeatJob {
4360                    prompt: fix_prompt.clone(),
4361                    spec,
4362                    seat,
4363                    cwd: wt.clone(),
4364                    timeout,
4365                    allow_write: true,
4366                    sessions,
4367                    artifacts: artifacts.clone(),
4368                    stem: "graft".to_owned(),
4369                    handover: None,
4370                }
4371            })
4372            .await;
4373        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4374            // Preserve the need to restore earlier fallback edits without
4375            // confusing an unstarted seat with an interrupted invocation.
4376            self.state.graft.as_mut().expect("graft record").outcome =
4377                GraftOutcome::WaitingForRestore;
4378            return self.stop_for_resource();
4379        }
4380        let mut why: Option<String> = None;
4381        match &out {
4382            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4383                why = Some("the fixer produced nothing usable".to_owned());
4384            }
4385            AgentOutcome::Ok(_) => {}
4386            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4387            AgentOutcome::Quota(o) => {
4388                self.state.quota.push(QuotaLoss {
4389                    seat: seat.key.clone(),
4390                    node: "graft".to_owned(),
4391                    at: Timestamp::now(),
4392                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4393                });
4394                why = Some("rate limited (quota)".to_owned());
4395            }
4396            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4397                why = Some(format!(
4398                    "the fixer failed: {}",
4399                    e.lines()
4400                        .next()
4401                        .unwrap_or("")
4402                        .chars()
4403                        .take(160)
4404                        .collect::<String>()
4405                ));
4406            }
4407        }
4408        let agent_id = seat.agent.clone();
4409        self.state.seats.insert(seat.key.clone(), seat);
4410
4411        let mut post = None;
4412        let mut lines = None;
4413        let _coordination = self.coordinate(&wt).await?;
4414        if why.is_none() {
4415            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4416            if let Ok(r) = &rescued {
4417                self.state.note_withheld("graft", &r.withheld);
4418            }
4419            let head = git::rev_parse(&wt, "HEAD").await?;
4420            // What is measured must be what gets reviewed and gated: a failed
4421            // rescue, or anything still uncommitted (withheld lockfiles
4422            // aside, which are never part of the tree), rejects the graft.
4423            let dirty = match &rescued {
4424                Ok(r) => git::status_porcelain(&wt)
4425                    .await
4426                    .map(|st| {
4427                        st.lines().any(|l| {
4428                            !r.withheld
4429                                .iter()
4430                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4431                        })
4432                    })
4433                    .unwrap_or(true),
4434                Err(_) => true,
4435            };
4436            if dirty {
4437                why = Some("the fixer left changes that could not be committed".to_owned());
4438            } else if head == pre {
4439                why = Some("the fixer changed nothing".to_owned());
4440            } else {
4441                lines = git::changed_lines(&wt, &pre, &head).await;
4442                post = Some(head);
4443            }
4444        }
4445        let (adoption, adoption_error) = match &out {
4446            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4447            _ => (None, None),
4448        };
4449        let requested = graft::requested(&plan);
4450        let outcome = match why {
4451            None => {
4452                self.state.event(
4453                    "graft",
4454                    format!(
4455                        "applied: {}",
4456                        lines.map_or_else(
4457                            || "changed lines unavailable".to_owned(),
4458                            |n| format!("{n} changed lines")
4459                        )
4460                    ),
4461                );
4462                GraftOutcome::Applied
4463            }
4464            Some(why) => {
4465                // Not settled until the restore is confirmed: the record
4466                // stays `Running`, so a resume retries it.
4467                crate::recovery::restore_repair(&wt, &pre)
4468                    .await
4469                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4470                self.state.event(
4471                    "graft",
4472                    format!("rolled back: {why}; the winner is untouched"),
4473                );
4474                GraftOutcome::RolledBack(why)
4475            }
4476        };
4477        self.state.graft = Some(GraftRecord {
4478            pre,
4479            post,
4480            lines,
4481            limit: None,
4482            agent: Some(agent_id),
4483            outcome,
4484            requested,
4485            adoption,
4486            adoption_error,
4487        });
4488        self.state.save()?;
4489        Ok(())
4490    }
4491
4492    // ----------------------------------------------------------------- fold
4493
4494    async fn fold_losers(&mut self) -> Result<()> {
4495        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4496            return Ok(());
4497        };
4498        if !self.state.candidate_preservation.is_empty() {
4499            bail!("candidate preservation incomplete; refusing cleanup");
4500        }
4501        let repo = self.state.repo.clone();
4502        let _coordination = self.coordinate(&repo).await?;
4503        let mut folded = Vec::new();
4504        for i in 0..self.state.candidates.len() {
4505            let c = &self.state.candidates[i];
4506            if c.label == winner || c.folded {
4507                continue;
4508            }
4509            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4510            remove_preserved_worktree(&repo, &wt).await?;
4511            if git::branch_exists(&repo, &branch).await?
4512                && !git::branch_delete(&repo, &branch).await?
4513            {
4514                bail!("candidate branch removal refused");
4515            }
4516            self.state.candidates[i].folded = true;
4517            self.state.save()?;
4518            folded.push(label.to_string());
4519        }
4520        // The judges are finished; their checkouts are pure cost from here.
4521        let root = self.state.worktree_root();
4522        for j in 1..=self.roles.judges.len() {
4523            let wt = root.join(format!("judge-{j}"));
4524            if wt.exists() {
4525                remove_preserved_worktree(&repo, &wt).await?;
4526            }
4527        }
4528        // The design-deliberation stage is finished by the time a tally
4529        // exists — same reasoning as the judges above.
4530        if self.state.config.graph.advise {
4531            for k in 1..=self.state.config.graph.advisors {
4532                let wt = root.join(format!("advisor-{k}"));
4533                if wt.exists() {
4534                    remove_preserved_worktree(&repo, &wt).await?;
4535                }
4536            }
4537        }
4538        if !folded.is_empty() {
4539            self.state
4540                .event("fold", format!("folded candidates {}", folded.join(", ")));
4541            self.state.save()?;
4542        }
4543        Ok(())
4544    }
4545
4546    // ------------------------------------------------------------ base sync
4547
4548    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4549    /// anything verifies it.
4550    ///
4551    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4552    /// read whatever is checked out in the winner's worktree. Left alone that
4553    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4554    /// it when the run *branched* - and a run takes long enough that the base
4555    /// has usually moved by the time it gets here. A gate that ran there
4556    /// answers "green on the commit this run started from", not "green on
4557    /// what is about to land", and the difference showed up three times in
4558    /// one day as a green run whose merge would have reverted a file another
4559    /// pull request had already landed.
4560    ///
4561    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4562    /// `land::Step::Rebase` calls, rather than a second implementation of the
4563    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4564    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4565    /// doc warns about ("two answers to a question nobody notices until a
4566    /// diff is wrong").
4567    ///
4568    /// A conflict is not the end of the road: the standing rebase is handed
4569    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4570    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4571    /// land). Once it finishes, review and the gate run as usual on the
4572    /// rebased tree, which is where a breakage the new base caused is caught
4573    /// by the ordinary gate-fix round. magi resolves nothing itself.
4574    ///
4575    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4576    /// `state.base_sync.attempts`, is how many times the base is *rebased
4577    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4578    /// *conflict* was given to a fixer. When the fixer cannot finish the
4579    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4580    /// what was tried (rounds spent, paths still conflicted) and the branch
4581    /// and worktree stay exactly as they were - untouched, for a person to
4582    /// look at - which is also what makes re-entering this function
4583    /// afterwards a no-op instead of a second attempt at the same wall. A
4584    /// push failure ends the same way.
4585    async fn sync_to_base(&mut self) -> Result<()> {
4586        if self.state.status == RunStatus::AlreadyInBase {
4587            return Ok(());
4588        }
4589        let conflicted = self
4590            .state
4591            .base_sync
4592            .as_ref()
4593            .is_some_and(|s| s.conflict.is_some());
4594        let Some(winner) = self.state.winner().cloned() else {
4595            return Ok(());
4596        };
4597
4598        let repo = self.state.repo.clone();
4599        let remote = self.state.config.merge.remote.clone();
4600        let base_branch = self.state.base_branch.clone();
4601        let tracking = format!("{remote}/{base_branch}");
4602
4603        git::fetch(&repo, &remote, &base_branch).await.ok();
4604        // No network, or the remote never had this branch: the run already
4605        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4606        // otherwise), and one that got this far is not blocked by a fetch
4607        // that fails now. It just has no newer tip to compare against.
4608        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4609            return Ok(());
4610        };
4611
4612        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4613        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4614        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4615
4616        // Before any rebase, and before a recorded conflict is honoured: a
4617        // branch whose change reached the base under other commit ids has
4618        // nothing to rebase and nothing to conflict with, and a run that
4619        // already stopped on that phantom conflict recovers here on resume.
4620        // `behind == 0` with head == tip is a branch the base has since taken
4621        // in whole, whether or not a conflict was ever recorded: the ancestry
4622        // proof must still run (`classify` ignores a head still on the start
4623        // commit).
4624        if (behind > 0 || conflicted || head == tip)
4625            && self
4626                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4627                .await?
4628        {
4629            return Ok(());
4630        }
4631        if conflicted {
4632            return Ok(());
4633        }
4634
4635        if behind == 0 {
4636            // A fixer-finished rebase moves the branch ref before the
4637            // winner's worktree is told (`sync_to_head` below). A run that
4638            // died in between resumes here with `behind == 0` and a tree still
4639            // holding the pre-rebase files, which review and the gate would
4640            // then read. That state is exactly: HEAD moved off the tip the
4641            // rebase started from, yet the tree is still identical to that
4642            // tip. A tree with edits of its own differs from it, so nothing
4643            // is thrown away.
4644            if let Some(from) = self
4645                .state
4646                .rebase_fixes
4647                .iter()
4648                .rev()
4649                .find_map(|r| r.from.clone())
4650                && from != head
4651                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4652                    .await
4653                    .is_ok_and(|o| o.ok())
4654            {
4655                git::sync_to_head(&winner.worktree).await?;
4656            }
4657            // An earlier attempt may have rebased the branch locally and died
4658            // before pushing it (only the fresh-rebase arm below pushes).
4659            // Publish it now, so the plain push at PR time is not refused as
4660            // a non-fast-forward. A run already holding a recorded conflict
4661            // never reaches here; that case is out of scope.
4662            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4663            if let Some(why) = &conflict {
4664                self.state.status = RunStatus::Blocked;
4665                self.state.event("land", why.clone());
4666            }
4667            self.state.base_sync = Some(BaseSync {
4668                tip,
4669                behind: 0,
4670                attempts,
4671                conflict,
4672                already_in: None,
4673            });
4674            self.state.save()?;
4675            return Ok(());
4676        }
4677
4678        if attempts >= BASE_SYNC_ROUNDS {
4679            let why = format!(
4680                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4681                 rebase(s); rebasing again would only race it",
4682                winner.branch
4683            );
4684            self.state.status = RunStatus::Blocked;
4685            self.state.base_sync = Some(BaseSync {
4686                tip,
4687                behind,
4688                attempts,
4689                conflict: Some(why.clone()),
4690                already_in: None,
4691            });
4692            self.state.event("land", why);
4693            self.state.save()?;
4694            return Ok(());
4695        }
4696
4697        self.state.event(
4698            "land",
4699            format!(
4700                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4701                winner.branch
4702            ),
4703        );
4704        self.state.save()?;
4705
4706        // The remote's copy of the branch, read now and only if the fetch
4707        // really succeeded (a stale tracking ref must never pin a lease). It is
4708        // pushed over after a rebase only when it is a commit this branch
4709        // already contains, by ancestry or by patch (an earlier rebase of ours
4710        // that never reached the remote): anything else is somebody else's work.
4711        let branch_tracking = format!("{remote}/{}", winner.branch);
4712        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4713        let remote_tip = if self.state.task_pr_branch().is_none()
4714            && matches!(&fetched_branch, Ok(o) if o.ok())
4715        {
4716            git::rev_parse(&repo, &branch_tracking).await.ok()
4717        } else {
4718            None
4719        };
4720        // A remote tip this branch does not contain is somebody else's work:
4721        // rebasing would leave a local tip that can never be pushed. Stop
4722        // before touching anything and say so.
4723        if let Some(theirs) = &remote_tip
4724            && !git::is_ancestor(&repo, theirs, &head).await
4725            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4726                .await
4727                .is_ok_and(|missing| missing.is_empty())
4728        {
4729            let why = format!(
4730                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4731                 them",
4732                short(theirs),
4733                winner.branch
4734            );
4735            self.state.status = RunStatus::Blocked;
4736            self.state.base_sync = Some(BaseSync {
4737                tip,
4738                behind,
4739                attempts,
4740                conflict: Some(why.clone()),
4741                already_in: None,
4742            });
4743            self.state.event("land", why);
4744            self.state.save()?;
4745            return Ok(());
4746        }
4747
4748        let scratch = self.state.dir().join("base-sync");
4749        let rebased = match crate::rebase::rebase_with_fixer(
4750            &mut self.state,
4751            &scratch,
4752            &winner.branch,
4753            &tracking,
4754        )
4755        .await
4756        {
4757            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4758            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4759            Err(e) => Err(e),
4760        };
4761        let attempts = attempts + 1;
4762        match rebased {
4763            Ok(None) => {
4764                // The branch ref moved, but a worktree that already had it
4765                // checked out (the winner's) was not told; sync its index and
4766                // files before anything reads them.
4767                git::sync_to_head(&winner.worktree).await?;
4768                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4769                let mut conflict = None;
4770                if let Some(pinned) = &remote_tip {
4771                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4772                    match pushed {
4773                        Ok(o) if o.ok() => self.state.event(
4774                            "land",
4775                            format!("pushed rebased {} to {remote}", winner.branch),
4776                        ),
4777                        Ok(o) => {
4778                            conflict = Some(format!(
4779                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4780                                winner.branch,
4781                                short(pinned),
4782                                o.stderr.chars().take(600).collect::<String>()
4783                            ));
4784                        }
4785                        Err(e) => {
4786                            conflict = Some(format!(
4787                                "rebased {} locally but could not push it: {e:#}",
4788                                winner.branch
4789                            ));
4790                        }
4791                    }
4792                }
4793                if let Some(why) = &conflict {
4794                    self.state.status = RunStatus::Blocked;
4795                    self.state.event("land", why.clone());
4796                }
4797                self.state.base_sync = Some(BaseSync {
4798                    tip: tip.clone(),
4799                    behind: 0,
4800                    attempts,
4801                    conflict,
4802                    already_in: None,
4803                });
4804                self.state
4805                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4806            }
4807            Ok(Some(conflict)) => {
4808                let why = format!(
4809                    "{} conflicts with {tracking} and did not rebase: {}",
4810                    winner.branch,
4811                    conflict.chars().take(600).collect::<String>()
4812                );
4813                self.state.status = RunStatus::Blocked;
4814                self.state.base_sync = Some(BaseSync {
4815                    tip,
4816                    behind,
4817                    attempts,
4818                    conflict: Some(why.clone()),
4819                    already_in: None,
4820                });
4821                self.state.event("land", why);
4822            }
4823            // A refused Git write or a busy repository: nothing is wrong with
4824            // the branch. The scratch rebase stays standing and the run
4825            // resumes into it (`base_sync` must not claim a conflict).
4826            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4827                self.state.save()?;
4828                return Ok(());
4829            }
4830            Err(e) => {
4831                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4832                self.state.status = RunStatus::Blocked;
4833                self.state.base_sync = Some(BaseSync {
4834                    tip,
4835                    behind,
4836                    attempts,
4837                    conflict: Some(why.clone()),
4838                    already_in: None,
4839                });
4840                self.state.event("land", why);
4841            }
4842        }
4843        self.state.save()?;
4844        Ok(())
4845    }
4846
4847    /// Push a branch an earlier attempt rebased locally but never published,
4848    /// pinned to the remote tip read right after a successful fetch. Returns
4849    /// the reason when the run must stop; `None` when there was nothing to do
4850    /// (no remote copy, the same tip, or a remote copy this branch already
4851    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4852    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4853        // Task competitions publish only after the PR text gate, to their
4854        // public head; never resurrect a candidate branch on the remote.
4855        if self.state.task_pr_branch().is_some() {
4856            return None;
4857        }
4858        let repo = self.state.repo.clone();
4859        let remote = self.state.config.merge.remote.clone();
4860        let fetched = git::fetch(&repo, &remote, branch).await;
4861        if !matches!(&fetched, Ok(o) if o.ok()) {
4862            return None;
4863        }
4864        let branch_tracking = format!("{remote}/{branch}");
4865        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4866        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4867            return None;
4868        }
4869        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4870            .await
4871            .is_ok_and(|missing| missing.is_empty())
4872        {
4873            return Some(format!(
4874                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4875                 them",
4876                short(&theirs)
4877            ));
4878        }
4879        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4880            Ok(o) if o.ok() => {
4881                self.state
4882                    .event("land", format!("pushed rebased {branch} to {remote}"));
4883                None
4884            }
4885            Ok(o) => Some(format!(
4886                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4887                 someone may have pushed): {}",
4888                short(&theirs),
4889                o.stderr.chars().take(600).collect::<String>()
4890            )),
4891            Err(e) => Some(format!(
4892                "{branch} is rebased locally but could not be pushed: {e:#}"
4893            )),
4894        }
4895    }
4896
4897    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4898    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4899    /// whether it did.
4900    ///
4901    /// Checked only when the base is ahead of the branch. A failing check is
4902    /// "not proven" - the ordinary rebase path then decides - never a reason to
4903    /// stop the run.
4904    ///
4905    /// The remote copy of the branch is held to the same standard as the local
4906    /// one: if it carries a tip this worktree does not, that tip must itself be
4907    /// proven in the base, or nothing is settled (a pull request would
4908    /// otherwise be closed over commits nobody checked). The pull request is
4909    /// closed *before* the terminal status is saved; if that fails for a
4910    /// reason other than a refusal (no network, a `gh` error) the run is left
4911    /// `Blocked` with the reason as its conflict, which a resume retries -
4912    /// the same recovery a phantom conflict gets.
4913    async fn settle_already_in(
4914        &mut self,
4915        branch: &str,
4916        tip: &str,
4917        head: &str,
4918        attempts: usize,
4919        behind: usize,
4920    ) -> Result<bool> {
4921        let repo = self.state.repo.clone();
4922        let remote = self.state.config.merge.remote.clone();
4923        let start = self.state.base_commit.clone();
4924        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4925            Ok(Some(e)) => e,
4926            Ok(None) => return Ok(false),
4927            Err(e) => {
4928                tracing::warn!("already-in-base check for {branch}: {e:#}");
4929                return Ok(false);
4930            }
4931        };
4932        let mut verified = vec![head.to_owned()];
4933        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4934        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4935        if matches!(&fetched, Ok(o) if o.ok())
4936            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4937            && theirs != head
4938        {
4939            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4940                Ok(Some(_)) => verified.push(theirs),
4941                _ => return Ok(false),
4942            }
4943        }
4944        let base_branch = self.state.base_branch.clone();
4945        let message = format!(
4946            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4947             land",
4948            evidence.names(),
4949            evidence.proof.as_str()
4950        );
4951        let closed =
4952            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4953                .await;
4954        match closed {
4955            Ok(Ok(url)) => self
4956                .state
4957                .event("land", format!("closed {url}: superseded on {base_branch}")),
4958            Ok(Err(why)) => self
4959                .state
4960                .event("land", format!("did not close a pull request: {why}")),
4961            Err(e) => {
4962                let why = format!(
4963                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4964                     not be closed ({e:#}); resume to retry"
4965                );
4966                self.state.status = RunStatus::Blocked;
4967                self.state.base_sync = Some(BaseSync {
4968                    tip: tip.to_owned(),
4969                    behind,
4970                    attempts,
4971                    conflict: Some(why.clone()),
4972                    already_in: None,
4973                });
4974                self.state.event("land", why);
4975                self.state.save()?;
4976                return Ok(true);
4977            }
4978        }
4979        self.state.status = RunStatus::AlreadyInBase;
4980        self.state.base_sync = Some(BaseSync {
4981            tip: tip.to_owned(),
4982            behind,
4983            attempts,
4984            conflict: None,
4985            already_in: Some(evidence),
4986        });
4987        self.state.event("land", message);
4988        self.state.save()?;
4989        self.settle_questions();
4990        Ok(true)
4991    }
4992
4993    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
4994    /// last landed the winner on, once it has run, else the commit the run
4995    /// branched from.
4996    ///
4997    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
4998    /// and `vote` all happen before there is a winner to rebase, so they
4999    /// compare every candidate against the branch point on purpose, and a
5000    /// base that moves after they are already done cannot change an answer
5001    /// they already gave.
5002    fn landing_base(&self) -> String {
5003        self.state
5004            .base_sync
5005            .as_ref()
5006            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5007    }
5008
5009    // ------------------------------------------------------- operator fix
5010
5011    /// Route specific, already-recorded review findings to a fixer for a
5012    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5013    /// entry point.
5014    ///
5015    /// Distinct from `review_loop`'s own fix step in three ways: it never
5016    /// runs a reviewer wave, it never spends review-round budget, and what
5017    /// happened is recorded as an [`OperatorFixRequest`] appended to
5018    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5019    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5020    /// and vote must never be rewritten to look like a manufactured blocking
5021    /// verdict.
5022    ///
5023    /// Only meaningful once review has actually concluded: `Ready` (handed
5024    /// off with findings still open, or simply concluded clean while minor
5025    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5026    /// gate failed). Everything else is refused: a run still in progress
5027    /// should simply be resumed, and a `Merged` run's branch has already
5028    /// landed — reopening *this* run's own record cannot change that, so the
5029    /// answer there is a fresh `magi review <branch>`.
5030    ///
5031    /// A real commit here re-verifies through a fresh, ordinary review-only
5032    /// run on the same branch ([`Self::review`]) rather than reopening this
5033    /// run's own `review_loop`: once any round in this run's history went
5034    /// clean, `review_conclusion` treats that as permanent by design (the
5035    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5036    /// way to force one more genuine reviewer wave out of *this* run without
5037    /// either rewriting history or weakening that guarantee for every other
5038    /// caller. A review-only run costs nothing extra — no implementation, no
5039    /// judging, no vote — and exercises the exact same review → verify →
5040    /// gate → (human) merge path, unmodified.
5041    pub async fn fix_selected(
5042        &mut self,
5043        ids: &[String],
5044        reason: &str,
5045        allow_stale: bool,
5046    ) -> Result<()> {
5047        let reason = reason.trim();
5048        if reason.is_empty() {
5049            bail!("a fix request needs a reason — that is the operator's own record of why");
5050        }
5051        if ids.is_empty() {
5052            bail!("no finding id given");
5053        }
5054        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5055            bail!(
5056                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5057                 has already concluded — can be given a targeted fix. A run still \
5058                 in progress should simply be resumed; a `merged` run's branch has \
5059                 already landed, so its answer is a fresh `magi review <branch>`, \
5060                 not reopening this run's own record",
5061                self.state.id,
5062                self.state.status.as_str()
5063            );
5064        }
5065        let Some(winner) = self.state.winner().cloned() else {
5066            bail!("run {} has no winning candidate to fix", self.state.id);
5067        };
5068        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5069            bail!(
5070                "branch `{}` no longer exists; this run cannot be extended",
5071                winner.branch
5072            );
5073        }
5074        let home = crate::run::home();
5075        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5076            bail!(
5077                "run {} is currently being worked on by another magi process",
5078                self.state.id
5079            );
5080        }
5081        // Held for the rest of this call, including the follow-up review
5082        // below: two `magi fix` invocations against the same run must not
5083        // both reach the worktree manipulation further down, which would
5084        // otherwise race to remove and recreate the same directory — see
5085        // [`FixClaim`]'s own doc.
5086        let _claim = FixClaim::acquire(&self.state.dir())?;
5087
5088        // Resolve every id before spending anything — an unknown id refuses
5089        // the whole request rather than silently dropping it — and dedup
5090        // while keeping the operator's own order.
5091        let mut seen = BTreeSet::new();
5092        let mut findings = Vec::new();
5093        let mut missing = Vec::new();
5094        for id in ids {
5095            if !seen.insert(id.clone()) {
5096                continue;
5097            }
5098            match self.state.finding(id) {
5099                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5100                    id: f.id.clone(),
5101                    severity: f.severity,
5102                    reviewer_vote: rec.vote,
5103                    round: round.round,
5104                    round_head: round.head.clone(),
5105                    reviewer: rec.reviewer,
5106                    agent: rec.agent.clone(),
5107                    file: f.file.clone(),
5108                    line: f.line,
5109                    title: f.title.clone(),
5110                    detail: f.detail.clone(),
5111                    outcome: OperatorFixOutcome::Pending,
5112                }),
5113                None => missing.push(id.clone()),
5114            }
5115        }
5116        if !missing.is_empty() {
5117            bail!(
5118                "unknown finding id(s): {}; nothing was changed",
5119                missing.join(", ")
5120            );
5121        }
5122
5123        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5124        let stale_details: Vec<(String, String)> = findings
5125            .iter()
5126            .filter(|f| f.round_head != head_at_request)
5127            .map(|f| (f.id.clone(), f.round_head.clone()))
5128            .collect();
5129        let stale = !stale_details.is_empty();
5130        if stale && !allow_stale {
5131            bail!(
5132                "the branch has moved since some finding(s) were raised — {} — now \
5133                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5134                stale_details
5135                    .iter()
5136                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5137                    .collect::<Vec<_>>()
5138                    .join(", "),
5139                short(&head_at_request)
5140            );
5141        }
5142
5143        let request = OperatorFixRequest {
5144            requested_at: Timestamp::now(),
5145            reason: reason.to_owned(),
5146            findings,
5147            head_at_request: head_at_request.clone(),
5148            allow_stale,
5149            stale,
5150            fix: None,
5151            result_head: None,
5152            follow_up_review_run: None,
5153        };
5154        self.state.event(
5155            "fix",
5156            format!(
5157                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5158                request.findings.len(),
5159                request
5160                    .findings
5161                    .iter()
5162                    .map(|f| f.id.as_str())
5163                    .collect::<Vec<_>>()
5164                    .join(", "),
5165            ),
5166        );
5167        // Recorded now, before any worktree work or the fixer call itself —
5168        // and re-saved at each checkpoint below: a crash at any point after
5169        // this (mid fixer call, mid follow-up review) must not lose the fact
5170        // that this was requested, for which findings, and why. Everything
5171        // past this point reads and writes through `request_index` rather
5172        // than a local variable, since `request` itself is moved here.
5173        self.state.operator_fixes.push(request);
5174        self.state.save()?;
5175        let request_index = self.state.operator_fixes.len() - 1;
5176
5177        // A fresh, dedicated worktree for this one call, never the winner's
5178        // own worktree in place: that one may already be gone (folded away),
5179        // and reusing it in place would leave the branch checked out there
5180        // when the follow-up review below tries to check it out again. Freed
5181        // immediately after, either way — but only once confirmed clean:
5182        // `worktree_remove` is a `git worktree remove --force`, which would
5183        // otherwise discard uncommitted work left there by the operator or
5184        // another process before this had a chance to even look at it.
5185        let repo = self.state.repo.clone();
5186        let coordination = self.coordinate(&repo).await?;
5187        if winner.worktree.exists() {
5188            // Lockfiles a rescue commit withheld stay untracked on purpose and
5189            // are already recorded; they are not the operator's work to protect.
5190            let dirty = git::git(
5191                &winner.worktree,
5192                &["status", "--porcelain", "--untracked-files=all"],
5193            )
5194            .await?;
5195            let only_withheld = dirty.lines().all(|l| {
5196                l.strip_prefix("?? ")
5197                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5198            });
5199            if !only_withheld {
5200                bail!(
5201                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5202                     discard them first",
5203                    winner.worktree.display()
5204                );
5205            }
5206            git::worktree_remove(&self.state.repo, &winner.worktree)
5207                .await
5208                .ok();
5209        }
5210        let fix_worktree = self.state.worktree_root().join("operator-fix");
5211        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5212        git::git(
5213            &self.state.repo,
5214            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5215        )
5216        .await
5217        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5218        if !git::is_clean(&fix_worktree).await? {
5219            git::worktree_remove(&self.state.repo, &fix_worktree)
5220                .await
5221                .ok();
5222            bail!(
5223                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5224                winner.branch
5225            );
5226        }
5227        let repo_for_setup = self.state.repo.clone();
5228        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5229            Ok(()) => {
5230                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5231            }
5232            Err(e) => Err(e),
5233        };
5234        if let Err(e) = setup {
5235            git::worktree_remove(&self.state.repo, &fix_worktree)
5236                .await
5237                .ok();
5238            return Err(e);
5239        }
5240
5241        let run_id = self.state.id.clone();
5242        let prompts = self.state.config.prompts.clone();
5243        let language = self.state.config.graph.language.clone();
5244        let sessions = self.state.config.graph.sessions;
5245        let artifacts = agent::artifacts_dir(&self.state.dir());
5246        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5247            .findings
5248            .iter()
5249            .map(|f| Finding {
5250                id: f.id.clone(),
5251                severity: f.severity,
5252                file: f.file.clone(),
5253                line: f.line,
5254                title: f.title.clone(),
5255                detail: f.detail.clone(),
5256            })
5257            .collect();
5258        let fix_prompt = prompt::operator_fix_in(
5259            &self.state.instruction,
5260            &finding_list,
5261            reason,
5262            &stale_details,
5263            &head_at_request,
5264            &language,
5265            &self.state.config.graph.github_language,
5266        );
5267        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5268        drop(coordination);
5269        let (job, seat, out) = self
5270            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5271                prompt: fix_prompt.clone(),
5272                spec,
5273                seat,
5274                cwd: fix_worktree.clone(),
5275                timeout,
5276                allow_write: true,
5277                sessions,
5278                artifacts: artifacts.clone(),
5279                stem: "operator-fix".to_owned(),
5280                handover: None,
5281            })
5282            .await;
5283        let agent_id = seat.agent.clone();
5284
5285        let mut fix = FixRecord {
5286            agent: agent_id,
5287            addressed: Vec::new(),
5288            rejected: Vec::new(),
5289            notes: String::new(),
5290            committed: false,
5291            failed: None,
5292            duration_ms: 0,
5293            continuation: None,
5294        };
5295        let mut final_seat = seat.clone();
5296        match out {
5297            AgentOutcome::Ok(o) => {
5298                fix.duration_ms = o.duration_ms;
5299                let parsed = verdict::extract_json::<FixReport>(&o.text);
5300                let incomplete_reason = match &parsed {
5301                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5302                        "the reply parsed, but it reported a command whose own CLI \
5303                         never confirmed an exit status"
5304                            .to_owned(),
5305                    ),
5306                    Ok(_) => None,
5307                    Err(e) => Some(e.to_string()),
5308                };
5309                match incomplete_reason {
5310                    None => {
5311                        let report = parsed.expect("checked Ok above");
5312                        fix.addressed = report.addressed;
5313                        fix.rejected = report.rejected;
5314                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5315                    }
5316                    Some(reason) => {
5317                        let (resumed_seat, resolved, failure, cont) = self
5318                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5319                            .await;
5320                        fix.duration_ms += cont.cumulative_wait_ms;
5321                        fix.continuation = Some(cont);
5322                        final_seat = resumed_seat;
5323                        match resolved {
5324                            Some(report) => {
5325                                fix.addressed = report.addressed;
5326                                fix.rejected = report.rejected;
5327                                fix.notes =
5328                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5329                            }
5330                            None => fix.failed = failure,
5331                        }
5332                    }
5333                }
5334            }
5335            AgentOutcome::Dropped(o) => {
5336                fix.duration_ms = o.duration_ms;
5337                let why = o
5338                    .dropped
5339                    .as_ref()
5340                    .map(|d| d.why.as_str())
5341                    .unwrap_or("the CLI ended the stream without delivering its answer");
5342                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5343            }
5344            AgentOutcome::Quota(o) => {
5345                self.state.quota.push(QuotaLoss {
5346                    seat: final_seat.key.clone(),
5347                    node: "fix".to_owned(),
5348                    at: Timestamp::now(),
5349                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5350                });
5351                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5352            }
5353            AgentOutcome::Failed(e) => fix.failed = Some(e),
5354            AgentOutcome::ResourceBlocked(e) => {
5355                fix.failed = Some(format!("the fixer was not started: {e}"));
5356            }
5357        }
5358        if fix.continuation.is_none() {
5359            fix.continuation = Some(ContinuationRecord::not_needed());
5360        }
5361        self.state.seats.insert(final_seat.key.clone(), final_seat);
5362
5363        let rescue_message = format!(
5364            "magi: operator-selected fix ({}) (uncommitted work)",
5365            self.state.operator_fixes[request_index]
5366                .findings
5367                .iter()
5368                .map(|f| f.id.as_str())
5369                .collect::<Vec<_>>()
5370                .join(", ")
5371        );
5372        let repo = self.state.repo.clone();
5373        let coordination = self.coordinate(&repo).await?;
5374        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5375            self.state.note_withheld("fix", &r.withheld);
5376        }
5377        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5378        fix.committed = after != head_at_request;
5379        git::worktree_remove(&self.state.repo, &fix_worktree)
5380            .await
5381            .ok();
5382        drop(coordination);
5383
5384        self.state.event(
5385            "fix",
5386            match &fix.failed {
5387                Some(reason) => format!(
5388                    "operator fix: adoption report was lost ({reason}); {}",
5389                    if fix.committed {
5390                        "committed"
5391                    } else {
5392                        "NO new commit"
5393                    }
5394                ),
5395                None => format!(
5396                    "operator fix: {} addressed, {} rejected, {}",
5397                    fix.addressed.len(),
5398                    fix.rejected.len(),
5399                    if fix.committed {
5400                        "committed"
5401                    } else {
5402                        "NO new commit"
5403                    }
5404                ),
5405            },
5406        );
5407
5408        // Every selected finding gets an outcome — never left `Pending` once
5409        // the fixer's own turn is over. A report that never came back at all
5410        // marks every one of them `Unreported`, not silently "not addressed":
5411        // quota, a dropped stream, or an exhausted continuation are gaps in
5412        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5413        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5414        for f in &mut self.state.operator_fixes[request_index].findings {
5415            f.outcome = if fix.failed.is_some() {
5416                OperatorFixOutcome::Unreported
5417            } else if fix.addressed.contains(&f.id) {
5418                OperatorFixOutcome::Addressed
5419            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5420                OperatorFixOutcome::Rejected { why: r.why.clone() }
5421            } else {
5422                OperatorFixOutcome::Unreported
5423            };
5424        }
5425
5426        let committed = fix.committed;
5427        if committed {
5428            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5429        }
5430        self.state.operator_fixes[request_index].fix = Some(fix);
5431        // Saved again now that the fixer's own outcome is final, on top of
5432        // the save right after the request was first pushed above.
5433        self.state.save()?;
5434
5435        if committed {
5436            self.state.event(
5437                "fix",
5438                format!(
5439                    "operator fix committed {}; opening a follow-up review-only run",
5440                    short(&after)
5441                ),
5442            );
5443            // The operator asked for the fix, and the follow-up serves whatever
5444            // task the run it follows served.
5445            let origin =
5446                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5447            match Self::review(
5448                &self.state.repo,
5449                &winner.branch,
5450                self.state.config.clone(),
5451                origin,
5452            )
5453            .await
5454            {
5455                Ok(mut follow_up) => {
5456                    follow_up.state.event(
5457                        "start",
5458                        format!(
5459                            "requested by an operator fix on run {} for finding(s) {}",
5460                            self.state.id,
5461                            self.state.operator_fixes[request_index]
5462                                .findings
5463                                .iter()
5464                                .map(|f| f.id.as_str())
5465                                .collect::<Vec<_>>()
5466                                .join(", "),
5467                        ),
5468                    );
5469                    follow_up.state.save()?;
5470                    let follow_up_id = follow_up.state.id.clone();
5471                    // The follow-up is a run like any other: it belongs to the
5472                    // task of the run it follows, or to one filed for it.
5473                    let adopted = match crate::direct::adopt(
5474                        &follow_up.state,
5475                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5476                    ) {
5477                        Ok(a) => a,
5478                        Err(e) => {
5479                            // An ownerless run must not spend agent calls; it
5480                            // stays saved, and `magi run --resume` adopts it.
5481                            self.state.event(
5482                                "fix",
5483                                format!(
5484                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5485                                ),
5486                            );
5487                            self.state.save()?;
5488                            return Ok(());
5489                        }
5490                    };
5491                    let executed = follow_up.execute().await;
5492                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5493                    if let Some(a) = adopted {
5494                        a.finish(&follow_up.state, executed);
5495                    }
5496                    if let Some(e) = failure {
5497                        self.state.event(
5498                            "fix",
5499                            format!(
5500                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5501                            ),
5502                        );
5503                    }
5504                    self.state.operator_fixes[request_index].follow_up_review_run =
5505                        Some(follow_up_id);
5506                }
5507                Err(e) => {
5508                    self.state.event(
5509                        "fix",
5510                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5511                    );
5512                }
5513            }
5514            self.state.save()?;
5515        }
5516
5517        Ok(())
5518    }
5519
5520    // --------------------------------------------------------------- review
5521
5522    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5523    /// names, in order, each at most once, moving on only when the call
5524    /// advances (an error, a quota hit or nothing usable - the same decision
5525    /// point `agent::chain_advances` is for the other chained roles).
5526    ///
5527    /// Each agent gets its own seat from `Runner::seat`: the same agent
5528    /// continues its conversation (the winner's own implementer seat when it
5529    /// is the winner's author, now that the competition is over), another
5530    /// takes a fresh one so the full prompt is sent again. A handover is
5531    /// recorded under `node`, which is what makes the fallback stick for the
5532    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5533    /// is returned, so an exhausted chain reads exactly like a single failed
5534    /// fixer: one quota loss, the same wording, the same refund. An earlier
5535    /// attempt's edits are left in the tree and judged, with the final
5536    /// attempt's, by what git says afterwards.
5537    ///
5538    /// Returns the job that produced the outcome, for `continue_fix_report`.
5539    async fn ask_fixer(
5540        &mut self,
5541        winner: &Candidate,
5542        node: &'static str,
5543        round: Option<usize>,
5544        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5545    ) -> (SeatJob, SeatState, AgentOutcome) {
5546        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5547        self.ask_fixer_among(attempts, node, round, None, build)
5548            .await
5549    }
5550
5551    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5552    /// worktree and a commit: before the next agent is asked after one that
5553    /// did not satisfy the call, the worktree is put back on that commit, so
5554    /// a failed agent's partial edits are not carried into the next one.
5555    async fn ask_fixer_among(
5556        &mut self,
5557        attempts: Vec<(AgentSpec, String)>,
5558        node: &'static str,
5559        round: Option<usize>,
5560        restore: Option<(&Path, &str)>,
5561        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5562    ) -> (SeatJob, SeatState, AgentOutcome) {
5563        let run_id = self.state.id.clone();
5564        let prompts = self.state.config.prompts.clone();
5565        let cache = self.state.config.cache_dir();
5566        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5567        let mut last = None;
5568        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5569            let seat = self.seat(&key, &spec.id);
5570            let mut job = build(spec.clone(), seat);
5571            if i > 0 {
5572                job.stem = format!("{}-{}", job.stem, spec.id);
5573            }
5574            let ctx = WaveCtx {
5575                carry_seats: false,
5576                pauses: Some((&self.pause, &self.interrupt)),
5577                run: &run_id,
5578                node,
5579                prompts: &prompts,
5580                cache: cache.as_deref(),
5581                round,
5582            };
5583            let (seat, out) =
5584                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5585            // A resource block never advances the chain: the chosen agent
5586            // was not asked, so it must be the one asked again on resume.
5587            let advances = match &out {
5588                AgentOutcome::Ok(o) => agent::output_advances(o),
5589                AgentOutcome::ResourceBlocked(_) => false,
5590                _ => true,
5591            };
5592            if let Some(next) = ids.get(i + 1)
5593                && advances
5594            {
5595                self.state.seats.insert(seat.key.clone(), seat.clone());
5596                let class =
5597                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5598                let reason = match &out {
5599                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5600                    other => fail_reason(other),
5601                };
5602                // A failed agent's edits must not reach the next one, and a
5603                // restore that did not work ends the chain: the call fails
5604                // and the caller's own (confirmed) restore decides.
5605                if let Some((wt, rev)) = restore
5606                    && let Err(e) = self.graft_restore(wt, rev).await
5607                {
5608                    let why =
5609                        format!("could not restore the worktree before the next fixer: {e:#}");
5610                    tracing::warn!("{why}");
5611                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5612                        AgentOutcome::ResourceBlocked(why)
5613                    } else {
5614                        AgentOutcome::Failed(why)
5615                    };
5616                    last = Some((job, seat, out));
5617                    break;
5618                }
5619                record_handover(
5620                    &mut self.state,
5621                    node,
5622                    &seat.key,
5623                    &spec.id,
5624                    next,
5625                    &class,
5626                    &reason,
5627                );
5628                continue;
5629            }
5630            last = Some((job, seat, out));
5631            break;
5632        }
5633        last.expect("the fixer chain always has an entry")
5634    }
5635
5636    /// The one repair a verification-only failure earns (see
5637    /// [`crate::recovery`]), before the first review round of a run that took
5638    /// a branch over.
5639    ///
5640    /// Saved as started *before* the fixer is called, so a resume never
5641    /// repairs twice; a start found on re-entry is rolled back to the commit
5642    /// it began from. A failed or unusable fixer leaves the tree as it was and
5643    /// the review goes on over it, so the first round's own verification
5644    /// reports the failure again. The failure is never carried into a round as
5645    /// a result: verification is re-run on whatever commit this leaves.
5646    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5647        use crate::recovery::{Mode, Repair};
5648        if !self.state.reviews.is_empty() {
5649            return Ok(());
5650        }
5651        let Some(mut rec) = self.state.recovery.clone() else {
5652            return Ok(());
5653        };
5654        if rec.mode != Mode::VerifyRepair {
5655            return Ok(());
5656        }
5657        let wt = winner.worktree.clone();
5658        let mut head = git::rev_parse(&wt, "HEAD").await?;
5659        match rec.repair.clone() {
5660            Repair::Done { .. } => return Ok(()),
5661            Repair::WaitingForRestore { before } => {
5662                self.graft_restore(&wt, &before).await?;
5663                head = before;
5664                rec.repair = Repair::NotStarted;
5665                self.state.recovery = Some(rec.clone());
5666                self.state.save()?;
5667            }
5668            Repair::Started { before } => {
5669                // Interrupted: the unfinished edits are not trusted.
5670                // An unconfirmed rollback blocks the run with `Started` kept,
5671                // so a resume retries it; the tree is never reviewed as is.
5672                if let Err(e) = self.graft_restore(&wt, &before).await {
5673                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5674                        return Err(e);
5675                    }
5676                    return self
5677                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5678                        .await;
5679                }
5680                rec.repair = Repair::Done {
5681                    before: before.clone(),
5682                    after: before,
5683                    committed: false,
5684                    failed: Some("interrupted; rolled back".to_owned()),
5685                };
5686                self.state
5687                    .event("recovery", "the repair was interrupted and rolled back");
5688                self.state.recovery = Some(rec);
5689                self.state.save()?;
5690                return Ok(());
5691            }
5692            Repair::NotStarted => {}
5693        }
5694        if crate::recovery::recheck(&mut rec, &head) {
5695            self.state.event("recovery", rec.summary());
5696            self.state.recovery = Some(rec);
5697            self.state.save()?;
5698            return Ok(());
5699        }
5700        rec.repair = Repair::Started {
5701            before: head.clone(),
5702        };
5703        self.state.recovery = Some(rec.clone());
5704        self.state.save()?;
5705
5706        let language = self.state.config.graph.language.clone();
5707        let sessions = self.state.config.graph.sessions;
5708        let artifacts = agent::artifacts_dir(&self.state.dir());
5709        let fix_prompt = prompt::recovery_fix(
5710            &self.state.instruction,
5711            &rec,
5712            self.state.config.graph.review_rounds,
5713            &language,
5714        );
5715        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5716        self.state.event(
5717            "recovery",
5718            format!(
5719                "repair: the fixer starts from the failed verification on {}",
5720                short(&head)
5721            ),
5722        );
5723        let (_job, seat, out) = self
5724            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5725                prompt: fix_prompt.clone(),
5726                spec,
5727                seat,
5728                cwd: wt.clone(),
5729                timeout,
5730                allow_write: true,
5731                sessions,
5732                artifacts: artifacts.clone(),
5733                stem: "fix-recovery".to_owned(),
5734                handover: None,
5735            })
5736            .await;
5737        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5738            rec.repair = Repair::WaitingForRestore { before: head };
5739            self.state.recovery = Some(rec);
5740            return self.stop_for_resource();
5741        }
5742        let failed = match &out {
5743            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5744            AgentOutcome::Quota(o) => {
5745                self.state.quota.push(QuotaLoss {
5746                    seat: seat.key.clone(),
5747                    node: "fix".to_owned(),
5748                    at: Timestamp::now(),
5749                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5750                });
5751                Some("rate limited (quota); the fixer could not run".to_owned())
5752            }
5753            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5754            AgentOutcome::Failed(e) => Some(e.clone()),
5755        };
5756        self.state.seats.insert(seat.key.clone(), seat);
5757        let coordination = self.coordinate(&wt).await?;
5758        let failed = match failed {
5759            // The agent's partial edits are not kept.
5760            Some(why) => {
5761                if let Err(e) = git::restore_to(&wt, &head).await {
5762                    return self
5763                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5764                        .await;
5765                }
5766                Some(why)
5767            }
5768            None => {
5769                if let Ok(r) = git::rescue_commit(
5770                    &wt,
5771                    "magi: recovery repair of the failed verification (uncommitted work)",
5772                )
5773                .await
5774                {
5775                    self.state.note_withheld("fix", &r.withheld);
5776                }
5777                None
5778            }
5779        };
5780        let after = git::rev_parse(&wt, "HEAD").await?;
5781        drop(coordination);
5782        self.state.event(
5783            "recovery",
5784            match &failed {
5785                Some(why) => format!("repair: nothing kept ({why})"),
5786                None if after != head => {
5787                    format!("repair: committed {}", short(&after))
5788                }
5789                None => "repair: the fixer made no commit".to_owned(),
5790            },
5791        );
5792        rec.repair = Repair::Done {
5793            before: head.clone(),
5794            committed: after != head,
5795            after,
5796            failed,
5797        };
5798        self.state.recovery = Some(rec);
5799        self.state.save()?;
5800        Ok(())
5801    }
5802
5803    /// A repair whose rollback could not be confirmed: the run is blocked and
5804    /// resumable, and the unconfirmed state is never reviewed.
5805    async fn stop_recovery(
5806        &mut self,
5807        mut rec: crate::recovery::RecoveryRecord,
5808        head: &str,
5809        why: String,
5810    ) -> Result<()> {
5811        // `Started` stays recorded: a resume retries the restore.
5812        rec.repair = crate::recovery::Repair::Started {
5813            before: head.to_owned(),
5814        };
5815        self.state.recovery = Some(rec);
5816        self.state
5817            .event("recovery", format!("repair could not be undone: {why}"));
5818        self.state.status = RunStatus::Blocked;
5819        self.state.save()?;
5820        bail!("recovery repair could not be rolled back: {why}")
5821    }
5822
5823    async fn review_loop(&mut self) -> Result<()> {
5824        // A base that would not rebase is a person's decision, not a review
5825        // round: nothing here would change the answer, and reviewers and a
5826        // fixer would be spending real budget on a tree that cannot land
5827        // regardless of what they find.
5828        if self
5829            .state
5830            .base_sync
5831            .as_ref()
5832            .is_some_and(|s| s.conflict.is_some())
5833        {
5834            return Ok(());
5835        }
5836        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5837        // agent files with `magi task add` name the run that paid for it. The
5838        // prompt overlay is cloned alongside it because the waves borrow it
5839        // while `self` is mutably borrowed by the node's own bookkeeping.
5840        let run_id = self.state.id.clone();
5841        let prompts = self.state.config.prompts.clone();
5842        let Some(winner) = self.state.winner().cloned() else {
5843            return Ok(());
5844        };
5845        let max_rounds = self.state.config.graph.review_rounds;
5846        // A clean round, an exhausted round budget, or a stalled tree (see
5847        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5848        // they are recorded — recomputed here, not read off `status`, so a
5849        // reentry into a run that already stopped restates the identical
5850        // verdict instead of silently handing back whatever an earlier node
5851        // in this same walk clobbered `status` to (a solo-candidate
5852        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5853        // below runs an empty range once the budget is spent, and would
5854        // otherwise fall through without touching `status` at all.
5855        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5856            // A reentry after a crash between the last round's save and
5857            // `stop_reviewing` reaches the hand-off here, not there.
5858            if status == RunStatus::Gating {
5859                self.record_contested_handoff();
5860            }
5861            self.state.status = status;
5862            self.state.save()?;
5863            return Ok(());
5864        }
5865        self.state.status = RunStatus::Reviewing;
5866        // A last recorded round whose own verification never resolved
5867        // (`ResourceBlocked` — the shared build cache, not the patch) is
5868        // never a concluded round, whatever the round budget says: starting
5869        // a fresh round on top of it would spend a whole new reviewer wave
5870        // re-reading an unchanged patch instead of just retrying the one
5871        // check that actually needs it, and once the budget is spent the
5872        // loop below has nothing left to do at all (its range is empty).
5873        // Retry that check directly instead, exactly the same retry
5874        // `stop_reviewing` already does for its own catch-up case.
5875        if self
5876            .state
5877            .reviews
5878            .last()
5879            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5880        {
5881            let shell = self.state.config.shell();
5882            return self
5883                .stop_reviewing(
5884                    "the last round's own verification never resolved",
5885                    &shell,
5886                    &winner.worktree,
5887                )
5888                .await;
5889        }
5890
5891        self.recovery_repair(&winner).await?;
5892        if self.resource_halt()? {
5893            return Ok(());
5894        }
5895
5896        let repo = self.state.repo.clone();
5897        let root = self.state.worktree_root();
5898        let language = self.state.config.graph.language.clone();
5899        let sessions = self.state.config.graph.sessions;
5900        let artifacts = self.merge_repair_artifacts();
5901        let base = self.landing_base();
5902        let base_short = short(&base);
5903        let reviewers = self.roles.reviewers.clone();
5904        let shell = self.state.config.shell();
5905
5906        for round in (self.state.reviews.len() + 1)..=max_rounds {
5907            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5908            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5909            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5910            // The prior round's own record, already persisted — never a
5911            // hand-carried variable of just its failing output: that is
5912            // exactly what let a round's e2e result drift out of sync with
5913            // which commit it was actually about (see `SCHEMA`'s doc for
5914            // schema 8). Judged against `head`, the commit reviewers are
5915            // about to look at now, so the summary always reads as "an
5916            // earlier head" here — this round's own patch has not been
5917            // checked yet.
5918            let prev_verification = self
5919                .state
5920                .reviews
5921                .last()
5922                .and_then(|r| r.verification_summary(&head));
5923            // Only the first round of a run that took a branch over: later
5924            // rounds have this run's own history.
5925            let prior_section = match (&self.state.recovery, round) {
5926                (Some(rec), 1) if self.state.reviews.is_empty() => {
5927                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5928                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5929                            Some(crate::recovery::head_of(
5930                                &git::diff(&winner.worktree, reviewed, "HEAD")
5931                                    .await
5932                                    .unwrap_or_default(),
5933                                crate::recovery::SINCE_DIFF_MAX,
5934                            ))
5935                        }
5936                        _ => None,
5937                    };
5938                    Some(prompt::recovery_review(rec, since.as_deref()))
5939                }
5940                _ => None,
5941            };
5942
5943            // Each reviewer gets its own detached checkout of exactly this
5944            // commit: nobody can perturb the winner's tree, and the fixer can
5945            // keep working without racing a reviewer.
5946            let mut jobs = Vec::new();
5947            let coordination = self.coordinate(&repo).await?;
5948            for (r, spec) in reviewers.iter().cloned().enumerate() {
5949                let wt = root.join(format!("review-{}", r + 1));
5950                // `clean -fdx` in a reset wipes setup's products, so every
5951                // round sets the seat up again.
5952                if wt.exists() {
5953                    git::reset_detached(&wt, &head).await?;
5954                } else {
5955                    git::worktree_add_detached(&repo, &wt, &head).await?;
5956                }
5957                self.setup_seat_worktree(&repo, &wt, false).await?;
5958                let seat_key = format!("review-{}", r + 1);
5959                // The seat starts the round on whoever answered it last, not
5960                // on the agent the spec names, so a failure is not re-paid.
5961                let spec = pick_start_spec(
5962                    &self.roles.reviewer_roster,
5963                    spec,
5964                    self.state.seat_history.get(&seat_key),
5965                );
5966                let seat = self.seat(&seat_key, &spec.id);
5967                jobs.push(SeatJob {
5968                    prompt: prompt::review(&prompt::ReviewCtx {
5969                        instruction: &self.state.instruction,
5970                        branch: &winner.branch,
5971                        base_short: &base_short,
5972                        stat: &stat,
5973                        patch: &patch,
5974                        verification: prev_verification.as_ref(),
5975                        prior: prior_section.as_deref(),
5976                        reviewers: reviewers.len(),
5977                        round,
5978                        rounds: max_rounds,
5979                        // A review-only run has no rankings, so nothing
5980                        // competed for this patch and the reviewer is told so.
5981                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5982                        lens: Lens::for_seat(r),
5983                        language: &language,
5984                        github_language: &self.state.config.graph.github_language,
5985                    }),
5986                    spec,
5987                    seat,
5988                    cwd: wt,
5989                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5990                    allow_write: false,
5991                    sessions,
5992                    artifacts: artifacts.clone(),
5993                    stem: format!("review-{round}-{}", r + 1),
5994                    handover: None,
5995                });
5996            }
5997
5998            drop(coordination);
5999            self.state.event(
6000                "review",
6001                format!(
6002                    "round {round}: {} reviewers on {}",
6003                    jobs.len(),
6004                    short(&head)
6005                ),
6006            );
6007            let mut quota_losses = Vec::new();
6008            let review_retries = self.state.config.graph.retries;
6009            let review_cache = self.state.config.cache_dir();
6010            let ctx = WaveCtx {
6011                carry_seats: true,
6012                pauses: Some((&self.pause, &self.interrupt)),
6013                run: &run_id,
6014                node: "review",
6015                prompts: &prompts,
6016                cache: review_cache.as_deref(),
6017                round: Some(round),
6018            };
6019            let results = ask_json_wave::<Review>(
6020                jobs,
6021                Arc::clone(&self.sem),
6022                review_retries,
6023                &self.roles.reviewer_roster,
6024                &ctx,
6025                &mut quota_losses,
6026                &mut self.state,
6027                &|_: &Review| Ok(()),
6028            )
6029            .await;
6030            if self.state.resource_block.is_some() {
6031                return self.stop_for_resource();
6032            }
6033            // Counted before the move below: how many of *this* round's
6034            // reviewer seats were lost to their own rate limit, as opposed to
6035            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6036            let round_quota_missing = quota_losses.len();
6037            self.state.quota.extend(quota_losses);
6038
6039            let mut records = Vec::new();
6040            let mut all_findings = Vec::new();
6041            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6042                let agent_id = seat.agent.clone();
6043                self.state.seats.insert(seat.key.clone(), seat);
6044                let mut record = ReviewRecord {
6045                    reviewer: r + 1,
6046                    agent: agent_id,
6047                    summary: String::new(),
6048                    findings: Vec::new(),
6049                    vote: None,
6050                    failed: None,
6051                    duration_ms: 0,
6052                    // Set for both outcomes: `failed: Some(_)` with
6053                    // `attempts > 0` is a seat every retry still lost, not a
6054                    // recovered one — only `failed: None` with `attempts > 0`
6055                    // reads as "answered after a nudge" (see this field's own
6056                    // doc).
6057                    attempts,
6058                };
6059                match res {
6060                    Ok((review, out)) => {
6061                        // Sanitized here, at the point every other piece of
6062                        // agent prose in this file is (candidate summaries,
6063                        // deliberation turns, vote reasons): a reviewer's own
6064                        // words are the one thing about it that could name
6065                        // it, and reconsideration below broadcasts this same
6066                        // summary and these same findings to every other
6067                        // seat on the panel.
6068                        record.summary =
6069                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6070                        record.vote = Some(review.vote);
6071                        record.duration_ms = out.duration_ms;
6072                        for (n, mut f) in review.findings.into_iter().enumerate() {
6073                            // ids are magi's, never the agent's: the fixer's
6074                            // adoption report is keyed by them.
6075                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6076                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6077                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6078                            // `file` is agent-supplied prose too, never
6079                            // checked against the real tree — the same
6080                            // exposure `title`/`detail` above have, just in
6081                            // a field easy to forget because it looks like a
6082                            // path rather than free text.
6083                            f.file = f
6084                                .file
6085                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6086                            all_findings.push(f.clone());
6087                            record.findings.push(f);
6088                        }
6089                        self.state.event(
6090                            "review",
6091                            format!(
6092                                "round {round}: reviewer {} voted {} with {} finding(s)",
6093                                r + 1,
6094                                review.vote.label(),
6095                                record.findings.len()
6096                            ),
6097                        );
6098                    }
6099                    Err(e) => {
6100                        record.failed = Some(e.to_string());
6101                        self.state.event(
6102                            "review",
6103                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6104                        );
6105                    }
6106                }
6107                records.push(record);
6108            }
6109
6110            // Tally the round's votes and, if they split, spend the one
6111            // round of reconsideration the split -> deliberate -> revote
6112            // shape `judge`/`vote` use for the panel, sized down to what a
6113            // read-only review round can afford: one round, and a revote
6114            // rather than an argument, because the panel already wrote its
6115            // reasoning down as findings the first time around.
6116            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6117            let vote_split =
6118                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6119            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6120            if vote_split {
6121                self.state.event(
6122                    "review",
6123                    format!(
6124                        "round {round}: votes split ({}) — one round of reconsideration",
6125                        initial_votes
6126                            .iter()
6127                            .map(|v| v.label())
6128                            .collect::<Vec<_>>()
6129                            .join(", ")
6130                    ),
6131                );
6132                // Seats read every seat's findings and votes, still numbered
6133                // and never named — the same anonymity `review` itself keeps.
6134                let panel: Vec<ReviewSeatReport<'_>> = records
6135                    .iter()
6136                    .filter_map(|r| {
6137                        r.vote.map(|vote| ReviewSeatReport {
6138                            reviewer: r.reviewer,
6139                            vote,
6140                            summary: &r.summary,
6141                            findings: &r.findings,
6142                        })
6143                    })
6144                    .collect();
6145
6146                let mut jobs = Vec::new();
6147                let mut seats_at = Vec::new();
6148                for (r, spec) in reviewers.iter().cloned().enumerate() {
6149                    // A seat with no initial vote has nothing to reconsider
6150                    // from and stays absent, the same as it stayed absent
6151                    // from `panel` above.
6152                    if records[r].vote.is_none() {
6153                        continue;
6154                    }
6155                    let wt = root.join(format!("review-{}", r + 1));
6156                    let seat_key = format!("review-{}", r + 1);
6157                    let spec = self.occupant(&seat_key, spec);
6158                    let seat = self.seat(&seat_key, &spec.id);
6159                    // A seat with no live session has already forgotten the
6160                    // initial review's prompt — restate the patch it is
6161                    // voting on, the same as `deliberate`/`vote` do for a
6162                    // judge in the same position.
6163                    // The panel already carries this seat's own review and
6164                    // vote, so restating the patch makes the prompt whole for
6165                    // a seat handed to another agent.
6166                    let build = |with_patch: bool| {
6167                        prompt::review_reconsider(&ReviewReconsiderCtx {
6168                            instruction: &self.state.instruction,
6169                            reviewer: r + 1,
6170                            lens: Lens::for_seat(r),
6171                            panel: &panel,
6172                            patch: with_patch.then_some(ReviewPatch {
6173                                branch: &winner.branch,
6174                                base_short: &base_short,
6175                                stat: &stat,
6176                                patch: &patch,
6177                            }),
6178                            round,
6179                            rounds: max_rounds,
6180                            language: &language,
6181                        })
6182                    };
6183                    let full = build(true);
6184                    let prompt = if has_context(&spec, &seat, sessions) {
6185                        build(false)
6186                    } else {
6187                        full.clone()
6188                    };
6189                    jobs.push(SeatJob {
6190                        prompt,
6191                        spec,
6192                        seat,
6193                        cwd: wt,
6194                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6195                        allow_write: false,
6196                        sessions,
6197                        artifacts: artifacts.clone(),
6198                        stem: format!("review-{round}-reconsider-{}", r + 1),
6199                        handover: Some(full),
6200                    });
6201                    seats_at.push(r);
6202                }
6203
6204                let mut recon_quota_losses = Vec::new();
6205                let recon_cache = self.state.config.cache_dir();
6206                let recon_ctx = WaveCtx {
6207                    carry_seats: true,
6208                    pauses: Some((&self.pause, &self.interrupt)),
6209                    run: &run_id,
6210                    node: "review",
6211                    prompts: &prompts,
6212                    cache: recon_cache.as_deref(),
6213                    round: Some(round),
6214                };
6215                let recon_results = ask_json_wave::<ReviewRevote>(
6216                    jobs,
6217                    Arc::clone(&self.sem),
6218                    review_retries,
6219                    &self.roles.reviewer_roster,
6220                    &recon_ctx,
6221                    &mut recon_quota_losses,
6222                    &mut self.state,
6223                    &|_: &ReviewRevote| Ok(()),
6224                )
6225                .await;
6226                if self.state.resource_block.is_some() {
6227                    return self.stop_for_resource();
6228                }
6229                self.state.quota.extend(recon_quota_losses);
6230
6231                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6232                    let agent_id = seat.agent.clone();
6233                    self.state.seats.insert(seat.key.clone(), seat);
6234                    let mut rec = ReviewRevoteRecord {
6235                        reviewer: r + 1,
6236                        agent: agent_id,
6237                        vote: None,
6238                        reason: String::new(),
6239                        failed: None,
6240                    };
6241                    match res {
6242                        Ok((rv, _)) => {
6243                            rec.vote = Some(rv.vote);
6244                            rec.reason =
6245                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6246                            self.state.event(
6247                                "review",
6248                                format!(
6249                                    "round {round}: reviewer {} revoted {}",
6250                                    r + 1,
6251                                    rv.vote.label()
6252                                ),
6253                            );
6254                        }
6255                        Err(e) => {
6256                            rec.failed = Some(e.to_string());
6257                            self.state.event(
6258                                "review",
6259                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6260                            );
6261                        }
6262                    }
6263                    reconsideration.push(rec);
6264                }
6265            } else if initial_votes.len() > 1 {
6266                self.state.event(
6267                    "review",
6268                    format!(
6269                        "round {round}: votes agreed ({}) — no reconsideration",
6270                        initial_votes[0].label()
6271                    ),
6272                );
6273            }
6274
6275            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6276            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6277            // A round that already has a blocking finding and a round left to
6278            // try is going back to the fixer no matter what `verify.e2e`
6279            // says, so running it first only spends the loop's slowest step
6280            // (minutes, for a Rust repo's full test suite) on a head about
6281            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6282            // runs once a round has no blocking findings left (see
6283            // `round_is_clean`, which a deferred — empty — `e2e` can never
6284            // satisfy since `blocking` is nonzero whenever this branch is
6285            // taken), and `stop_reviewing` forces a real run before it will
6286            // ever read a deferred round as green.
6287            let defer_e2e =
6288                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6289            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6290                let reason =
6291                    format!("{blocking} blocking finding(s) already required a fix this round");
6292                self.state.event(
6293                    "verify",
6294                    format!(
6295                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6296                         {}); it will run once a round has none left",
6297                        short(&head)
6298                    ),
6299                );
6300                (Vec::new(), false, true, Some(reason))
6301            } else {
6302                let e2e_commands = self.state.config.verify.e2e.clone();
6303                let cache_dir = self.state.config.cache_dir();
6304                let context = format!("round {round}");
6305                let wait = self.repository_wait();
6306                let (e2e, verify_retried) = with_cache_lease(
6307                    &mut self.state,
6308                    cache_dir.as_deref(),
6309                    "e2e",
6310                    "e2e",
6311                    &winner.worktree,
6312                    &head,
6313                    verify_timeout,
6314                    &context,
6315                    &wait,
6316                    |state, budget| {
6317                        let shell = shell.clone();
6318                        let e2e_commands = e2e_commands.clone();
6319                        let worktree = winner.worktree.clone();
6320                        let context = context.clone();
6321                        async move {
6322                            run_e2e_with_retry(
6323                                state,
6324                                &shell,
6325                                &e2e_commands,
6326                                &worktree,
6327                                budget,
6328                                &context,
6329                            )
6330                            .await
6331                        }
6332                    },
6333                )
6334                .await?;
6335                (e2e, verify_retried, false, None)
6336            };
6337
6338            let expected = records.len();
6339            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6340            let incomplete = answered < expected;
6341            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6342            let policy = self.state.config.graph.incomplete_review;
6343            let clean = round_is_clean(
6344                blocking,
6345                e2e_ok,
6346                answered,
6347                expected,
6348                round_quota_missing,
6349                policy,
6350            );
6351
6352            let mut round_record = ReviewRound {
6353                round,
6354                head: head.clone(),
6355                verified_head: None,
6356                verified_at: None,
6357                reviews: records,
6358                e2e,
6359                verify_retried,
6360                e2e_deferred,
6361                e2e_defer_reason,
6362                fix: None,
6363                blocking,
6364                answered,
6365                expected,
6366                clean,
6367                progressed: false,
6368                vote_split,
6369                reconsideration,
6370                verdict: None,
6371            };
6372            // The final vote per seat is its revote where reconsideration
6373            // ran and answered, its initial vote otherwise — the same
6374            // fallback `tally` uses for a judge whose private vote failed.
6375            round_record.verdict = ReviewVote::worst(
6376                round_record
6377                    .final_votes()
6378                    .into_iter()
6379                    .map(|(_, _, vote)| vote),
6380            );
6381            // Which commit and when magi actually attempted to check —
6382            // known the moment a command was dispatched against `head`,
6383            // whether or not it finished: a resource-blocked attempt still
6384            // targeted a specific commit at a specific time, and leaving
6385            // that unrecorded is exactly what made `verification_summary`
6386            // report a fresh attempt as "commit unknown ... recorded before
6387            // this was tracked", indistinguishable from a genuinely old,
6388            // untracked record. Only a deferred or unconfigured round never
6389            // ran at all and has nothing to record — see
6390            // `ReviewRound::verified_head`'s own doc.
6391            if !matches!(
6392                round_record.e2e_status(),
6393                E2eStatus::Deferred | E2eStatus::NotConfigured
6394            ) {
6395                round_record.verified_head = Some(head.clone());
6396                round_record.verified_at = Some(Timestamp::now());
6397            }
6398            let this_round_verification = round_record.verification_summary(&head);
6399
6400            if incomplete {
6401                let missing: Vec<String> = round_record
6402                    .reviews
6403                    .iter()
6404                    .filter(|r| r.failed.is_some())
6405                    .map(|r| format!("review-{}", r.reviewer))
6406                    .collect();
6407                self.state.event(
6408                    "review",
6409                    format!(
6410                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6411                        missing.join(", ")
6412                    ),
6413                );
6414            }
6415
6416            if clean {
6417                self.state.event(
6418                    "review",
6419                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6420                        format!(
6421                            "round {round}: clean (warn policy, incomplete panel) — no \
6422                             blocking findings from the seats that answered, verification green"
6423                        )
6424                    } else if incomplete {
6425                        format!(
6426                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6427                             quorum) — no blocking findings from the seats that answered, \
6428                             verification green",
6429                            expected - answered
6430                        )
6431                    } else {
6432                        format!("round {round}: clean — no blocking findings, verification green")
6433                    },
6434                );
6435                self.state.reviews.push(round_record);
6436                self.state.status = RunStatus::Gating;
6437                self.state.save()?;
6438                return Ok(());
6439            }
6440
6441            // Nothing was raised and verification passed, but not every seat
6442            // answered and `round_is_clean` still refused to call it clean —
6443            // either a seat is missing for a reason other than its own quota
6444            // (a crash, a timeout, unparsable output — worth another try), or
6445            // every seat that could have answered lost its quota and nobody
6446            // is left to decide on: re-review rather than send the fixer
6447            // after a round with nothing to fix.
6448            if incomplete && blocking == 0 && e2e_ok {
6449                self.state.reviews.push(round_record);
6450                self.state.save()?;
6451                if round == max_rounds {
6452                    self.state.status = RunStatus::Blocked;
6453                    self.state.event(
6454                        "review",
6455                        format!(
6456                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6457                             refusing to call it clean",
6458                            expected - answered
6459                        ),
6460                    );
6461                    return Ok(());
6462                }
6463                continue;
6464            }
6465
6466            // Nothing for the fixer to act on (`blocking == 0`) and the only
6467            // reason this round is not clean is that magi itself never got
6468            // a command to run — the shared build cache, not the patch (see
6469            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6470            // the fixer would invite a change to appease contention that has
6471            // nothing to do with the diff, and would leave this attempt
6472            // sitting in the next round's prompt as if it were about an
6473            // earlier, superseded commit rather than what it actually is:
6474            // the same head, still waiting to be checked. Wait for it the
6475            // same way the final round's own contention is already handled,
6476            // whatever round this happens to be.
6477            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6478                self.state.reviews.push(round_record);
6479                return self
6480                    .stop_reviewing(
6481                        "the round's own verification could not run",
6482                        &shell,
6483                        &winner.worktree,
6484                    )
6485                    .await;
6486            }
6487
6488            if round == max_rounds {
6489                self.state.reviews.push(round_record);
6490                return self
6491                    .stop_reviewing(
6492                        &format!(
6493                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6494                        ),
6495                        &shell,
6496                        &winner.worktree,
6497                    )
6498                    .await;
6499            }
6500
6501            // Fix. The winner's own implementer seat continues its conversation:
6502            // the competition is over, so context is pure benefit now.
6503            let blocking_findings: Vec<_> = all_findings
6504                .iter()
6505                .filter(|f| f.severity.blocks())
6506                .cloned()
6507                .collect();
6508            let fix_prompt = prompt::fix_in(
6509                &self.state.instruction,
6510                &blocking_findings,
6511                this_round_verification.as_ref(),
6512                round,
6513                max_rounds,
6514                &language,
6515                &self.state.config.graph.github_language,
6516            );
6517            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6518            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6519            let (job, seat, out) = self
6520                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6521                    prompt: fix_prompt.clone(),
6522                    spec,
6523                    seat,
6524                    cwd: winner.worktree.clone(),
6525                    timeout,
6526                    allow_write: true,
6527                    sessions,
6528                    artifacts: artifacts.clone(),
6529                    stem: format!("fix-{round}"),
6530                    handover: None,
6531                })
6532                .await;
6533            // Not asked: the round is not recorded (so a resume repeats it),
6534            // no fixer was handed over to, and the seat is left as it was.
6535            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6536                return self.stop_for_resource();
6537            }
6538            let agent_id = seat.agent.clone();
6539
6540            let mut fix = FixRecord {
6541                agent: agent_id,
6542                addressed: Vec::new(),
6543                rejected: Vec::new(),
6544                notes: String::new(),
6545                committed: false,
6546                failed: None,
6547                duration_ms: 0,
6548                continuation: None,
6549            };
6550            let mut continuation = ContinuationRecord::not_needed();
6551            let mut final_seat = seat.clone();
6552            match out {
6553                AgentOutcome::Ok(o) => {
6554                    fix.duration_ms = o.duration_ms;
6555                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6556                    // A parsed report standing next to a command this same
6557                    // reply's own CLI never confirmed the exit status of is
6558                    // not a resolved answer — the identical `CommandEvidence`
6559                    // `state.jobs` renders, read here instead of only on
6560                    // display, per the completion judgment and the shown
6561                    // record needing to agree.
6562                    let incomplete_reason = match &parsed {
6563                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6564                            "the reply parsed, but it reported a command whose own CLI never \
6565                             confirmed an exit status"
6566                                .to_owned(),
6567                        ),
6568                        Ok(_) => None,
6569                        Err(e) => Some(e.to_string()),
6570                    };
6571                    match incomplete_reason {
6572                        None => {
6573                            let report = parsed.expect("checked Ok above");
6574                            fix.addressed = report.addressed;
6575                            fix.rejected = report.rejected;
6576                            fix.notes =
6577                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6578                        }
6579                        Some(reason) => {
6580                            let (resumed_seat, resolved, failure, cont) = self
6581                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6582                                .await;
6583                            fix.duration_ms += cont.cumulative_wait_ms;
6584                            continuation = cont;
6585                            final_seat = resumed_seat;
6586                            match resolved {
6587                                Some(report) => {
6588                                    fix.addressed = report.addressed;
6589                                    fix.rejected = report.rejected;
6590                                    fix.notes = blind::sanitize_prose(
6591                                        &report.notes,
6592                                        &self.state.config.blind,
6593                                    );
6594                                }
6595                                None => fix.failed = failure,
6596                            }
6597                        }
6598                    }
6599                }
6600                // The CLI's raw error JSON is not a fix report to parse.
6601                AgentOutcome::Dropped(o) => {
6602                    fix.duration_ms = o.duration_ms;
6603                    let why = o
6604                        .dropped
6605                        .as_ref()
6606                        .map(|d| d.why.as_str())
6607                        .unwrap_or("the CLI ended the stream without delivering its answer");
6608                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6609                }
6610                AgentOutcome::Quota(o) => {
6611                    self.state.quota.push(QuotaLoss {
6612                        seat: final_seat.key.clone(),
6613                        node: "fix".to_owned(),
6614                        at: Timestamp::now(),
6615                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6616                    });
6617                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6618                }
6619                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6620            }
6621            fix.continuation = Some(continuation);
6622            self.state.seats.insert(final_seat.key.clone(), final_seat);
6623            let coordination = self.coordinate(&winner.worktree).await?;
6624            if let Ok(r) = git::rescue_commit(
6625                &winner.worktree,
6626                &format!("magi: review round {round} fixes (uncommitted work)"),
6627            )
6628            .await
6629            {
6630                self.state.note_withheld("fix", &r.withheld);
6631            }
6632            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6633            fix.committed = after != before;
6634            // Judged by what `git` says moved against base, never by the
6635            // fixer's own `addressed`/`rejected` count — see
6636            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6637            // `patch` snapshot above: swallowing this error would default
6638            // `diff_after` to empty, which almost always differs from a
6639            // non-empty `patch` and reads as "progressed" — exactly backwards
6640            // for a `git` failure the stagnation check cannot see through.
6641            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6642            drop(coordination);
6643            let progressed = diff_after != patch;
6644            let commit_note = if fix.committed {
6645                "committed"
6646            } else {
6647                "NO new commit"
6648            };
6649            let tree_note = if progressed {
6650                "changed vs base"
6651            } else {
6652                "unchanged vs base"
6653            };
6654            self.state.event(
6655                "fix",
6656                match &fix.failed {
6657                    // Distinct on purpose from "0 addressed, 0 rejected": the
6658                    // fixer's own diff still landed (blocking counts do keep
6659                    // falling round over round), only its adoption report did
6660                    // not come back, so this must never read like every
6661                    // finding was reviewed and declined.
6662                    Some(reason) => {
6663                        format!(
6664                            "round {round}: fixer's adoption report was lost ({reason}); \
6665                             {commit_note}, tree {tree_note}"
6666                        )
6667                    }
6668                    None => format!(
6669                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6670                         {tree_note}{}",
6671                        fix.addressed.len(),
6672                        fix.rejected.len(),
6673                        if continuation.outcome == ContinuationOutcome::Resumed {
6674                            format!(
6675                                " (adoption report recovered after {} continuation(s))",
6676                                continuation.attempts
6677                            )
6678                        } else {
6679                            String::new()
6680                        },
6681                    ),
6682                },
6683            );
6684            round_record.fix = Some(fix);
6685            round_record.progressed = progressed;
6686            self.state.reviews.push(round_record);
6687            self.state.save()?;
6688
6689            // The fixer's own report never came back this round, even after
6690            // `continue_fix_report`'s own budget was spent on it — not an
6691            // ordinary "no report" (dropped stream, quota, plain failure),
6692            // which already reads that way and is left to the existing round
6693            // budget. Stopping here, rather than opening another round, is
6694            // what keeps a next reviewer/fixer wave from ever being
6695            // dispatched onto `winner.worktree` while whatever the seat's
6696            // last call may still have running there is unaccounted for: no
6697            // process liveness check exists (and none is being added — see
6698            // AGENTS.md/this task's own scope), so the only way to honour
6699            // "nothing starts before a valid report returns" is to not start
6700            // anything further on this worktree from this run at all.
6701            if matches!(
6702                continuation.outcome,
6703                ContinuationOutcome::Exhausted
6704                    | ContinuationOutcome::QuotaLost
6705                    | ContinuationOutcome::NoSession
6706            ) {
6707                return self
6708                    .stop_reviewing(
6709                        "the fixer's adoption report never came back, even after resuming its \
6710                         own seat; refusing to start another round against the same worktree \
6711                         while that is unresolved",
6712                        &shell,
6713                        &winner.worktree,
6714                    )
6715                    .await;
6716            }
6717
6718            let streak = self
6719                .state
6720                .reviews
6721                .iter()
6722                .rev()
6723                .take_while(|r| !r.progressed)
6724                .count();
6725            if streak >= STAGNANT_LIMIT {
6726                return self
6727                    .stop_reviewing(
6728                        &format!(
6729                            "the tree has not moved against base for {streak} round(s) in a row"
6730                        ),
6731                        &shell,
6732                        &winner.worktree,
6733                    )
6734                    .await;
6735            }
6736        }
6737        Ok(())
6738    }
6739
6740    /// Decide, from the last recorded round's own verification, whether
6741    /// stopping the review loop is a hand-off or a genuine block.
6742    ///
6743    /// Called once the loop has given up trying — the round budget is spent,
6744    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6745    /// findings still open, never while a round is still clean or the
6746    /// incomplete-panel case handled inline above. Gate and e2e are facts
6747    /// about the tree; a lingering review finding is an opinion, and this
6748    /// workload's own `magi stats` puts reviewer precision low enough
6749    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6750    /// must not by itself stand between a green, verified change and the
6751    /// human who decides what to do with it. A red e2e is not an opinion, so
6752    /// that case still blocks, with the failing command and a tail of its
6753    /// output recorded here rather than left in `run.json` for someone to go
6754    /// find.
6755    ///
6756    /// A round that deferred its own e2e (see [`Config::graph`]'s
6757    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6758    /// only because nothing ran, and treating an empty list as a passing one
6759    /// here is exactly the "deferred painted green" bug this function exists
6760    /// to not have. When the last round's own verification never resolved —
6761    /// deferred on purpose, or a real attempt the shared build cache blocked
6762    /// — this makes (or retries) the real run, on the actual worktree this
6763    /// loop is about to stop touching, before deciding anything. A
6764    /// resource-blocked attempt is likewise never read as either green or
6765    /// red: it is evidence about the machine, not the patch (see
6766    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6767    /// blocked cache leaves this call without deciding rather than guessing
6768    /// — the caller retries on a later reentry.
6769    /// Record, once, that the review loop handed off over a blocking finding
6770    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6771    /// `land` asks the owner even with `land_approval` off. Called from every
6772    /// path that concludes `Gating`; a reentry keeps the first record.
6773    fn record_contested_handoff(&mut self) {
6774        if self.state.contested_handoff.is_some() {
6775            return;
6776        }
6777        let Some(contested) = self
6778            .state
6779            .reviews
6780            .last()
6781            .and_then(ReviewRound::contested_handoff)
6782        else {
6783            return;
6784        };
6785        self.state.event(
6786            "review",
6787            format!(
6788                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6789                 wait for the owner's approval",
6790                contested.findings.len(),
6791                contested.rejecters.len()
6792            ),
6793        );
6794        self.state.contested_handoff = Some(contested);
6795    }
6796
6797    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6798        let round_idx = self.state.reviews.len() - 1;
6799        // A deferred round and a resource-blocked one are the same shape
6800        // here: neither has a real result yet, and both get one more
6801        // attempt. Read off `e2e_status` — the single source for this —
6802        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6803        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6804        // still retries instead of being read as a settled result the
6805        // instant it stops being empty.
6806        let needs_catchup_run = matches!(
6807            self.state.reviews[round_idx].e2e_status(),
6808            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6809        );
6810        if needs_catchup_run {
6811            let round = self.state.reviews[round_idx].round;
6812            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6813            let commands = self.state.config.verify.e2e.clone();
6814            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6815            let cache_dir = self.state.config.cache_dir();
6816            let context = format!(
6817                "round {round}: verification unresolved, catching up before the final decision"
6818            );
6819            let wait = self.repository_wait();
6820            let (outcomes, verify_retried) = with_cache_lease(
6821                &mut self.state,
6822                cache_dir.as_deref(),
6823                "e2e",
6824                "e2e",
6825                worktree,
6826                &attempted_head,
6827                timeout,
6828                &context,
6829                &wait,
6830                |state, budget| {
6831                    let shell = shell.to_vec();
6832                    let commands = commands.clone();
6833                    let context = context.clone();
6834                    async move {
6835                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6836                            .await
6837                    }
6838                },
6839            )
6840            .await?;
6841            let last = &mut self.state.reviews[round_idx];
6842            last.e2e = outcomes;
6843            last.verify_retried = verify_retried;
6844            // Always the commit and time this attempt actually targeted,
6845            // whether or not it happens to equal the reviewed `head` and
6846            // whether or not a command finished — see
6847            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6848            // attempt is recorded too, so a later reader sees "attempted
6849            // again at T2" rather than silence.
6850            last.verified_head = Some(attempted_head);
6851            last.verified_at = Some(Timestamp::now());
6852            if verify_inconclusive(&last.e2e) {
6853                // Still not a real result: `e2e_deferred` is left exactly
6854                // as it was, so `needs_catchup_run` above reads
6855                // `ResourceBlocked` (via `e2e_status`, which checks
6856                // `resource_blocked` before `e2e_deferred`) and retries
6857                // again on the next reentry, rather than recording
6858                // contention as a red e2e and blocking the run on it.
6859                self.state.save()?;
6860                return Ok(());
6861            }
6862            last.e2e_deferred = false;
6863        }
6864        let last = &self.state.reviews[round_idx];
6865        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6866
6867        match last.e2e_status() {
6868            E2eStatus::Failed => {
6869                let red: Vec<String> = last
6870                    .e2e
6871                    .iter()
6872                    .filter(|o| !o.ok())
6873                    .map(|o| {
6874                        format!(
6875                            "`{}` -> {:?}\n{}",
6876                            o.command,
6877                            o.code,
6878                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6879                        )
6880                    })
6881                    .collect();
6882                self.state
6883                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6884                self.state.status = RunStatus::Blocked;
6885            }
6886            // `needs_catchup_run` above already retried once this call; if
6887            // it is still blocked, this is magi's own admission it could
6888            // not get a command to run, never a verdict on the patch — the
6889            // run is left exactly where a later reentry can retry again.
6890            E2eStatus::ResourceBlocked => {
6891                self.state.event(
6892                    "review",
6893                    format!(
6894                        "{why}; e2e could not run (shared build cache unavailable); not \
6895                         deciding yet"
6896                    ),
6897                );
6898            }
6899            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6900                self.state.event(
6901                    "review",
6902                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6903                );
6904                self.record_contested_handoff();
6905                self.state.status = RunStatus::Gating;
6906            }
6907        }
6908        self.state.save()?;
6909        Ok(())
6910    }
6911
6912    // ----------------------------------------------------------------- gate
6913
6914    async fn gate(&mut self) -> Result<()> {
6915        // Judged by the review record itself, not by `status`: a solo
6916        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6917        // reentry (see `judge`), and trusting it here is exactly how a run
6918        // that exhausted its review budget got gated and merged a second
6919        // time around. `review_conclusion` recomputes the review loop's own
6920        // verdict from the round records themselves — `Gating` for a clean
6921        // round or a hand-off (see `stop_reviewing`), anything else means the
6922        // loop is still going or genuinely blocked.
6923        // A base the winner could not be replayed onto is a decision, not a
6924        // round: there is no landing tree to gate. Read as its own record for
6925        // the same reason the review verdict is.
6926        if self.state.status == RunStatus::Failed
6927            || self
6928                .state
6929                .base_sync
6930                .as_ref()
6931                .is_some_and(|s| s.conflict.is_some())
6932            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6933                != Some(RunStatus::Gating)
6934        {
6935            return Ok(());
6936        }
6937        if self.state.gate_ran {
6938            // `review_loop` derives its conclusion from the clean review
6939            // record on every reentry and therefore puts a completed run back
6940            // in `Gating`. A recorded gate is a stronger, terminal fact:
6941            // retain its original command output (or lack of any, for a repo
6942            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6943            // doc) and restore `Blocked` on a real failure rather than
6944            // pretending the command is still running or running it a second
6945            // time. `gate_ran == false` remains the only shape — unattempted,
6946            // or a resource-blocked retry — that may still need to execute a
6947            // command.
6948            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6949                self.state.status = RunStatus::Blocked;
6950                self.state.save()?;
6951            }
6952            return Ok(());
6953        }
6954        let Some(winner) = self.state.winner().cloned() else {
6955            return Ok(());
6956        };
6957        self.state.status = RunStatus::Gating;
6958        let mut outcomes = self.run_gate(&winner).await?;
6959        loop {
6960            // A resource-blocked outcome means the gate command never actually
6961            // ran - the shared build cache could not be acquired or confirmed
6962            // fresh in time - which is evidence about the machine, not about
6963            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6964            // Recording it as a red gate would mark a run `Blocked` on nothing
6965            // but contention magi has already logged; leaving `self.state.gate`
6966            // empty and `self.state.gate_ran` false instead keeps the shape
6967            // this function already treats as "still needs to run" (see the
6968            // early-return above), so the next call retries the command
6969            // rather than concluding anything.
6970            if verify_inconclusive(&outcomes) {
6971                self.state.save()?;
6972                return Ok(());
6973            }
6974            if outcomes.iter().all(CommandOutcome::ok) {
6975                break;
6976            }
6977            match self.gate_fix_round(&winner, &outcomes).await? {
6978                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6979                GateFix::Stop => break,
6980                GateFix::Defer => {
6981                    self.state.save()?;
6982                    return Ok(());
6983                }
6984            }
6985        }
6986        let passed = outcomes.iter().all(CommandOutcome::ok);
6987        self.state.gate = outcomes;
6988        self.state.gate_ran = true;
6989        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6990        if !passed {
6991            self.state.status = RunStatus::Blocked;
6992            let spent = self.state.gate_fixes.len();
6993            self.state.event(
6994                "gate",
6995                if spent == 0 {
6996                    "gate failed; not merging".to_owned()
6997                } else {
6998                    format!("gate failed after {spent} gate-fix round(s); not merging")
6999                },
7000            );
7001        }
7002        self.state.save()?;
7003        Ok(())
7004    }
7005
7006    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7007    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7008    /// after review is clean and never on a candidate awaiting judging.
7009    ///
7010    /// Never fails the run: a non-zero exit or timeout is a warning and a
7011    /// recorded outcome, and the gate remains the single arbiter. Nothing
7012    /// configured means nothing happens - no event, no commit. `commit_all`
7013    /// commits any leftover change under the neutral identity and returns
7014    /// `false` when the tree is clean, so no empty commit is ever made.
7015    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7016        let commands = state.config.verify.pre_gate.clone();
7017        if commands.is_empty() {
7018            return;
7019        }
7020        let shell = state.config.shell();
7021        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7022        let (outcomes, _) = run_commands(
7023            state,
7024            "pre_gate",
7025            "pre_gate",
7026            0,
7027            &shell,
7028            &commands,
7029            &winner.worktree,
7030            timeout,
7031        )
7032        .await;
7033        for o in &outcomes {
7034            if !o.ok() {
7035                tracing::warn!(
7036                    "pre_gate `{}` failed ({:?}); the gate decides",
7037                    o.command,
7038                    o.code
7039                );
7040            }
7041            state.event(
7042                "pre_gate",
7043                format!(
7044                    "`{}` -> {}",
7045                    o.command,
7046                    if o.ok() {
7047                        "pass".to_owned()
7048                    } else {
7049                        format!(
7050                            "FAIL ({:?})\n{}",
7051                            o.code,
7052                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7053                        )
7054                    }
7055                ),
7056            );
7057        }
7058        state.pre_gate = outcomes;
7059        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7060            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7061                Ok(head) => {
7062                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7063                    state.pre_gate_commit = Some(head);
7064                }
7065                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7066            },
7067            Ok(false) => {}
7068            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7069        }
7070        if let Err(e) = state.save() {
7071            tracing::warn!("could not persist the pre_gate record: {e:#}");
7072        }
7073    }
7074
7075    /// Run `verify.gate` once against the winner's current tree, logging one
7076    /// event per command. Empty when nothing is configured.
7077    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7078        let wait = self.repository_wait();
7079        run_landing_gate(&mut self.state, winner, &wait).await
7080    }
7081
7082    /// One bounded fix round for a failing gate.
7083    ///
7084    /// The fixer is told the failure came from the gate itself, not from a
7085    /// reviewer, and is shown the failed commands, their exit codes and a tail
7086    /// of their output - whatever `[verify].gate` holds, nothing here knows
7087    /// what those commands run. Only a normal non-zero exit that printed
7088    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7089    /// command or a full disk says nothing about the code, and a fixer sent
7090    /// after it can only appease the machine. The round is judged by what git
7091    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7092    /// again before the gate does, so a fix cannot trade a green gate for a
7093    /// red e2e unnoticed.
7094    async fn gate_fix_round(
7095        &mut self,
7096        winner: &Candidate,
7097        outcomes: &[CommandOutcome],
7098    ) -> Result<GateFix> {
7099        let cap = self.state.config.graph.gate_fix_rounds;
7100        let spent = self.state.gate_fixes.len();
7101        if spent >= cap {
7102            if cap > 0 {
7103                self.state.event(
7104                    "gate",
7105                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7106                );
7107            }
7108            return Ok(GateFix::Stop);
7109        }
7110        if !gate_fixable(outcomes) {
7111            self.state.event(
7112                "gate",
7113                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7114                 command or similar); not spending a fix round on it",
7115            );
7116            return Ok(GateFix::Stop);
7117        }
7118        let min_free = self.state.config.disk.min_free_bytes;
7119        if min_free > 0 {
7120            match crate::disk::free_bytes(&winner.worktree) {
7121                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7122                Ok(free) => {
7123                    self.state.event(
7124                        "gate",
7125                        format!(
7126                            "only {free} bytes free ({min_free} required by `[disk] \
7127                             min_free_bytes`); not spending a fix round on a failure the disk \
7128                             may explain"
7129                        ),
7130                    );
7131                    return Ok(GateFix::Stop);
7132                }
7133                Err(e) => {
7134                    self.state.event(
7135                        "gate",
7136                        format!("free disk space could not be measured ({e:#}); no fix round"),
7137                    );
7138                    return Ok(GateFix::Stop);
7139                }
7140            }
7141        }
7142
7143        let attempt = spent + 1;
7144        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7145        let base = self.landing_base();
7146        let fix_prompt = prompt::gate_fix_in(
7147            &self.state.instruction,
7148            &failed,
7149            attempt,
7150            cap,
7151            &self.state.config.graph.language,
7152            &self.state.config.graph.github_language,
7153        );
7154        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7155        let sessions = self.state.config.graph.sessions;
7156        let artifacts = self.merge_repair_artifacts();
7157        self.state.event(
7158            "gate",
7159            format!("gate failed; gate-fix round {attempt} of {cap}"),
7160        );
7161        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7162        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7163        // Persist before invoking the fixer: it may commit and then be
7164        // interrupted before its result (or the following e2e) is saved.
7165        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7166            head: before.clone(),
7167            passed: false,
7168            outcomes: Vec::new(),
7169        });
7170        self.state.save()?;
7171        let (_, seat, out) = self
7172            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7173                prompt: fix_prompt.clone(),
7174                spec,
7175                seat,
7176                cwd: winner.worktree.clone(),
7177                timeout,
7178                allow_write: true,
7179                sessions,
7180                artifacts: artifacts.clone(),
7181                stem: format!("gate-fix-{attempt}"),
7182                handover: None,
7183            })
7184            .await;
7185        // The fixer was never asked: no round is spent (nothing is recorded)
7186        // and a later reentry asks the same agent again.
7187        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7188            return Ok(GateFix::Defer);
7189        }
7190        let mut record = GateFixRecord {
7191            agent: seat.agent.clone(),
7192            failed,
7193            notes: String::new(),
7194            committed: false,
7195            error: None,
7196        };
7197        match out {
7198            AgentOutcome::Ok(o) => {
7199                // A missing report is not a failed fix: the round is judged
7200                // by the tree below, and the report only carries prose.
7201                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7202                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7203                }
7204            }
7205            AgentOutcome::Dropped(_) => {
7206                record.error = Some("the CLI dropped the stream".to_owned());
7207            }
7208            AgentOutcome::Quota(o) => {
7209                self.state.quota.push(QuotaLoss {
7210                    seat: seat.key.clone(),
7211                    node: "gate-fix".to_owned(),
7212                    at: Timestamp::now(),
7213                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7214                });
7215                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7216            }
7217            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7218        }
7219        self.state.seats.insert(seat.key.clone(), seat);
7220        let coordination = self.coordinate(&winner.worktree).await?;
7221        if let Ok(r) = git::rescue_commit(
7222            &winner.worktree,
7223            &format!("magi: gate fix {attempt} (uncommitted work)"),
7224        )
7225        .await
7226        {
7227            self.state.note_withheld("gate-fix", &r.withheld);
7228        }
7229        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7230        record.committed = after != before;
7231        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7232        let note = record.error.clone();
7233        self.state.gate_fixes.push(record);
7234        self.state.save()?;
7235        if !changed {
7236            self.state.event(
7237                "gate",
7238                match note {
7239                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7240                    None => format!("gate-fix round {attempt}: the tree did not change"),
7241                },
7242            );
7243            return Ok(GateFix::Stop);
7244        }
7245        self.state.event(
7246            "gate",
7247            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7248        );
7249
7250        drop(coordination);
7251        let wait = self.repository_wait();
7252        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7253    }
7254
7255    // ---------------------------------------------------------------- merge
7256
7257    /// One read-only rewrite by the summary's author; a text the gate still
7258    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7259    /// applies on their word or on silence. `None` means the run is parked on
7260    /// that question and nothing may be pushed or opened yet.
7261    async fn guarded_pr_message(
7262        &mut self,
7263        winner: &Candidate,
7264        facts: Option<&BranchFacts>,
7265        posting: bool,
7266    ) -> Result<Option<PrMessage>> {
7267        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7268        let head = match final_head.as_deref() {
7269            Some(h) => crate::pr_outcome::FinalHead::At(h),
7270            None => crate::pr_outcome::FinalHead::Unknown,
7271        };
7272        // Only a pull request that is about to be posted pays for a summary.
7273        let summarized = if posting {
7274            Some(self.summarize_change(winner, final_head.as_deref()).await)
7275        } else {
7276            None
7277        };
7278        // A busy shared resource is not a permanent cause: publishing an
7279        // "unavailable" description would stick (a later head move never
7280        // edits the body). Stop before anything is pushed or opened; the
7281        // run is `Blocked` and a resume asks the summary again.
7282        if self.state.resource_block.is_some() {
7283            self.state.event(
7284                "pr-summary",
7285                "a shared resource was busy; nothing was published, resume to summarize again",
7286            );
7287            return Ok(None);
7288        }
7289        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7290            anyhow::bail!(
7291                "the final commit moved during public description generation; retry before publishing"
7292            );
7293        }
7294        let note = match (&summarized, &self.state.change_summary) {
7295            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7296            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7297            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7298            (None, _) => ChangeNote::NotAsked,
7299        };
7300        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7301        if !posting {
7302            let identity = crate::scrub::Identity::current();
7303            return Ok(Some(PrMessage {
7304                title: crate::scrub::scrub(&pr.title, &identity),
7305                body: crate::scrub::scrub(&pr.body, &identity),
7306            }));
7307        }
7308        // This very text was already rejected and asked about: no second
7309        // judge call, rewrite or question, just carry the record on.
7310        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7311        if let Some(known) = self
7312            .state
7313            .github_text
7314            .clone()
7315            .filter(|g| g.fingerprint == raw_fp)
7316        {
7317            return self.settle_withheld(winner, &pr, known).await;
7318        }
7319        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7320        let mut violations =
7321            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7322        let identity = crate::scrub::Identity::current();
7323        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7324            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7325            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7326        {
7327            violations.push(crate::github_text::Violation::SensitiveData);
7328        }
7329        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7330        // only a language violation starts one; sensitive data alone goes
7331        // straight to the owner, with the redacted text as the fallback.
7332        let has_language = violations
7333            .iter()
7334            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7335        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7336            self.state.event(
7337                "github-text",
7338                if has_language {
7339                    format!("description rejected: {violations:?}; requesting one rewrite")
7340                } else {
7341                    format!("description rejected: {violations:?}; asking the owner")
7342                },
7343            );
7344            let mut rewritten = false;
7345            let mut rejected_rewrite = None;
7346            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7347            {
7348                let output = text;
7349                if let Some(title) = summary_title(&output) {
7350                    // Judge the redacted candidate that will actually be posted.
7351                    // Sensitive data alone must not discard a useful rewrite.
7352                    let raw_body = summary_without_title(&output);
7353                    let clean_title = crate::scrub::scrub(&title, &identity);
7354                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7355                    let redacted = clean_title != title || body != raw_body;
7356                    let title = clean_title;
7357                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7358                    if !complete {
7359                        self.state.event(
7360                            "github-text",
7361                            "description rewrite omitted required findings",
7362                        );
7363                    }
7364                    if !body.trim().is_empty() {
7365                        body.push_str("\n\n");
7366                        body.push_str(&public_pr_evidence(&self.state, head));
7367                    }
7368                    let marker = format!("magi:run/{}", self.state.id);
7369                    if !body.trim().is_empty() && !body.contains(&marker) {
7370                        body.push_str(&format!("\n\n{marker}"));
7371                    }
7372                    let rewrite = PrMessage {
7373                        title: title.clone(),
7374                        body: body.clone(),
7375                    };
7376                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7377                    let mut failed = crate::github_text::check_for(
7378                        &self.state.config.graph,
7379                        &title,
7380                        &body,
7381                        redecision,
7382                    );
7383                    if (!crate::github_text::shareable(&title)
7384                        || !crate::github_text::shareable(&body))
7385                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7386                    {
7387                        failed.push(crate::github_text::Violation::SensitiveData);
7388                    }
7389                    if complete
7390                        && !title.trim().is_empty()
7391                        && !body.trim().is_empty()
7392                        && !failed.is_empty()
7393                    {
7394                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7395                            title: title.clone(),
7396                            body: body.clone(),
7397                            failed: failed
7398                                .iter()
7399                                .map(|v| crate::github_text::category(*v).to_owned())
7400                                .collect(),
7401                        });
7402                    }
7403                    if complete
7404                        && !title.trim().is_empty()
7405                        && !raw_body.trim().is_empty()
7406                        && failed.is_empty()
7407                    {
7408                        decision = redecision;
7409                        pr = PrMessage { title, body };
7410                        rewritten = true;
7411                        self.state
7412                            .event("github-text", "description rewrite passed");
7413                        if redacted {
7414                            self.state.event(
7415                                "github-text",
7416                                "sensitive data removed from description rewrite",
7417                            );
7418                        }
7419                    }
7420                }
7421            }
7422            if !rewritten {
7423                self.state.event(
7424                    "github-text",
7425                    "description rewrite unavailable or rejected; asking the owner",
7426                );
7427                if let Some(w) =
7428                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7429                {
7430                    // A different text than the one asked about before: that
7431                    // question no longer describes anything.
7432                    self.retire_github_text_question("the withheld text changed");
7433                    let record = crate::run::GithubTextAsk {
7434                        rewrite: rejected_rewrite,
7435                        fingerprint: raw_fp,
7436                        title: w.title,
7437                        body: w.body,
7438                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7439                        question: None,
7440                        asks: 0,
7441                        resolved: false,
7442                        chosen_title: None,
7443                        choice: None,
7444                    };
7445                    return self.settle_withheld(winner, &pr, record).await;
7446                }
7447            }
7448        }
7449        let (title, body) =
7450            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7451        Ok(Some(PrMessage { title, body }))
7452    }
7453
7454    /// Describe the whole change from its final diff in either run mode.
7455    ///
7456    /// The summary is tied to the final commit and to the merge base of the
7457    /// resolved remote base; a record for any other pair is dropped and the
7458    /// agent asked again, so a description never outlives the diff it was
7459    /// written from. The seat is read-only and the prompt tells it to read the
7460    /// diff and the code around it. `[roles] synthesizer` selects the agents
7461    /// (unset: the default pick), each asked at most once, advancing on an
7462    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7463    /// carries the reason it is unavailable; nothing is invented then.
7464    async fn summarize_change(
7465        &mut self,
7466        winner: &Candidate,
7467        head: Option<&str>,
7468    ) -> std::result::Result<(), String> {
7469        let Some(head) = head else {
7470            return Err("the final commit could not be read".to_owned());
7471        };
7472        let remote = self.state.config.merge.remote.clone();
7473        let tracking = format!("{remote}/{}", self.state.base_branch);
7474        if !git::rev_exists(&winner.worktree, &tracking).await {
7475            return Err(format!("the remote base `{tracking}` could not be read"));
7476        }
7477        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7478            Ok(b) if !b.is_empty() => b,
7479            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7480        };
7481        let evidence = crate::pr_outcome::summary_evidence(
7482            &self.state,
7483            crate::pr_outcome::FinalHead::At(head),
7484        );
7485        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7486        if self
7487            .state
7488            .change_summary
7489            .as_ref()
7490            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7491        {
7492            return Ok(());
7493        }
7494        if self.state.change_summary.take().is_some() {
7495            self.state.event(
7496                "pr-summary",
7497                "the final diff or evidence changed; the earlier change summary was discarded",
7498            );
7499        }
7500        // Save the source snapshot even if no generation agent is available.
7501        if let Err(e) = write_artifact(
7502            &self.state,
7503            &format!("public-description-{fingerprint}.md"),
7504            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7505        ) {
7506            tracing::warn!("could not save public description evidence: {e:#}");
7507        }
7508        let chain = match agent::pick_chain(
7509            &self.state.config.agents,
7510            self.state.config.roles.synthesizer.as_ref(),
7511            &agent::installed,
7512            "synthesizer",
7513        ) {
7514            Ok(chain) => chain,
7515            Err(e) => {
7516                tracing::warn!("no agent for the change summary: {e:#}");
7517                self.state.event("pr-summary", "no summary agent available");
7518                return Err("no agent was available to summarize it".to_owned());
7519            }
7520        };
7521        let subjects = git::subjects(&winner.worktree, &base, head)
7522            .await
7523            .unwrap_or_default();
7524        let stat = git::diff_stat(&winner.worktree, &base, head)
7525            .await
7526            .unwrap_or_default();
7527        let stat: String = stat.chars().take(6_000).collect();
7528        let mut prompt = crate::prompt::change_summary_in(
7529            &base,
7530            head,
7531            &subjects,
7532            stat.trim(),
7533            &self.state.config.graph.github_language,
7534        );
7535        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7536        let sessions = self.state.config.graph.sessions;
7537        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7538        let prompts = self.state.config.prompts.clone();
7539        let cache = self.state.config.cache_dir();
7540        let run = self.state.id.clone();
7541        for (i, spec) in chain.into_iter().enumerate() {
7542            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7543            let job = SeatJob {
7544                spec: spec.clone(),
7545                seat,
7546                cwd: winner.worktree.clone(),
7547                prompt: prompt.clone(),
7548                timeout,
7549                allow_write: false,
7550                sessions,
7551                artifacts: agent::artifacts_dir(&self.state.dir()),
7552                stem: if i == 0 {
7553                    "change-summary".to_owned()
7554                } else {
7555                    format!("change-summary-{}", spec.id)
7556                },
7557                handover: None,
7558            };
7559            let ctx = WaveCtx {
7560                run: &run,
7561                node: "pr-summary",
7562                prompts: &prompts,
7563                cache: cache.as_deref(),
7564                round: None,
7565                carry_seats: false,
7566                pauses: Some((&self.pause, &self.interrupt)),
7567            };
7568            let (seat, outcome) =
7569                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7570            self.state.seats.insert(seat.key.clone(), seat);
7571            let text = match outcome {
7572                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7573                AgentOutcome::ResourceBlocked(why) => {
7574                    // No agent was asked: the run stops resumably before
7575                    // anything is published (see `guarded_pr_message`).
7576                    if self.state.resource_block.is_none() {
7577                        self.state.resource_block = Some(crate::run::ResourceBlock {
7578                            node: "pr-summary".to_owned(),
7579                            kind: "shared resource".to_owned(),
7580                            detail: why,
7581                            at: Timestamp::now(),
7582                        });
7583                    }
7584                    return Err("a shared resource was busy".to_owned());
7585                }
7586                _ => continue,
7587            };
7588            let blind = &self.state.config.blind;
7589            let Some(title) = summary_title(&text) else {
7590                continue;
7591            };
7592            let body: String = summary_without_title(&text)
7593                .lines()
7594                .filter(|l| !l.contains("magi:run/"))
7595                .collect::<Vec<_>>()
7596                .join("\n");
7597            let body = blind::sanitize_prose(body.trim(), blind);
7598            let title = blind::sanitize_prose(&title, blind);
7599            if body.trim().is_empty() || title.trim().is_empty() {
7600                continue;
7601            }
7602            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7603                continue;
7604            }
7605            // The branch must still be the one the agent was shown.
7606            if git::rev_parse(&winner.worktree, "HEAD")
7607                .await
7608                .ok()
7609                .as_deref()
7610                != Some(head)
7611            {
7612                return Err("the branch moved while the summary was written".to_owned());
7613            }
7614            if git::merge_base(&winner.worktree, &tracking, head)
7615                .await
7616                .ok()
7617                .as_deref()
7618                != Some(base.as_str())
7619                || crate::pr_outcome::summary_evidence(
7620                    &self.state,
7621                    crate::pr_outcome::FinalHead::At(head),
7622                ) != evidence
7623            {
7624                return Err("the summary evidence changed while it was written".to_owned());
7625            }
7626            self.state.change_summary = Some(crate::run::ChangeSummary {
7627                head: head.to_owned(),
7628                base,
7629                evidence: fingerprint,
7630                title,
7631                body,
7632            });
7633            self.state.event(
7634                "pr-summary",
7635                format!("change summary written by {} for the final diff", spec.id),
7636            );
7637            if let Err(e) = self.state.save() {
7638                tracing::warn!("could not save the change summary: {e:#}");
7639            }
7640            return Ok(());
7641        }
7642        self.state
7643            .event("pr-summary", "no agent produced a change summary");
7644        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7645    }
7646
7647    /// One read-only rewrite request for a withheld description. The
7648    /// implementer's own conversation is reused only when it exists and can be
7649    /// resumed; otherwise the `[roles] language_judge` chain (or the default
7650    /// pick) writes it in a separate conversation. The prompt carries the whole
7651    /// text, so no agent needs any memory. Each agent is asked at most once and
7652    /// the chain advances on an error, quota, unusable answer or a reply with
7653    /// no `TITLE:` line. Nothing is recorded as a handover.
7654    async fn rewrite_pr_text(
7655        &mut self,
7656        winner: &Candidate,
7657        pr: &PrMessage,
7658        violations: &[crate::github_text::Violation],
7659    ) -> Option<String> {
7660        let prompt = format!(
7661            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7662            pr.title, pr.body
7663        );
7664        let sessions = self.state.config.graph.sessions;
7665        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7666        // (spec, seat, resumed)
7667        let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7668        let key = format!("impl-{}", winner.label);
7669        if let Some(spec) = self
7670            .state
7671            .config
7672            .agents
7673            .iter()
7674            .find(|a| a.id == winner.agent)
7675            .cloned()
7676            && let Some(existing) = self.state.seats.get(&key)
7677            && existing.agent == spec.id
7678            && agent::has_session(spec.kind, existing, sessions)
7679        {
7680            attempts.push((spec, existing.clone(), true));
7681        } else {
7682            let chain = agent::pick_chain(
7683                &self.state.config.agents,
7684                self.state.config.roles.language_judge.as_ref(),
7685                &agent::installed,
7686                "language_judge",
7687            );
7688            match chain {
7689                Ok(chain) => {
7690                    for spec in chain {
7691                        let seat = crate::fixer::seat_for(
7692                            &mut self.state,
7693                            "github-text-rewrite",
7694                            &spec.id,
7695                        );
7696                        attempts.push((spec, seat, false));
7697                    }
7698                }
7699                Err(e) => {
7700                    tracing::warn!("no agent for the description rewrite: {e:#}");
7701                    self.state
7702                        .event("github-text", "no rewrite agent available");
7703                    return None;
7704                }
7705            }
7706        }
7707        let prompts = self.state.config.prompts.clone();
7708        let cache = self.state.config.cache_dir();
7709        let run = self.state.id.clone();
7710        for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7711            let job = SeatJob {
7712                spec: spec.clone(),
7713                seat,
7714                cwd: winner.worktree.clone(),
7715                prompt: prompt.clone(),
7716                timeout: retry_budget(timeout_full, resumed),
7717                allow_write: false,
7718                sessions,
7719                artifacts: agent::artifacts_dir(&self.state.dir()),
7720                stem: if i == 0 {
7721                    "github-text-rewrite".to_owned()
7722                } else {
7723                    format!("github-text-rewrite-{}", spec.id)
7724                },
7725                handover: None,
7726            };
7727            let ctx = WaveCtx {
7728                run: &run,
7729                node: "github-text",
7730                prompts: &prompts,
7731                cache: cache.as_deref(),
7732                round: None,
7733                carry_seats: false,
7734                pauses: Some((&self.pause, &self.interrupt)),
7735            };
7736            let (seat, outcome) =
7737                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7738            self.state.seats.insert(seat.key.clone(), seat);
7739            match outcome {
7740                AgentOutcome::Ok(o)
7741                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7742                {
7743                    return Some(o.text);
7744                }
7745                AgentOutcome::ResourceBlocked(_) => return None,
7746                _ => {}
7747            }
7748        }
7749        None
7750    }
7751
7752    /// Abandon the standing posting-gate question, if any.
7753    fn retire_github_text_question(&mut self, why: &str) {
7754        let Some(id) = self
7755            .state
7756            .github_text
7757            .as_ref()
7758            .and_then(|g| g.question.clone())
7759        else {
7760            return;
7761        };
7762        let _ = ask::Questions::open().update(&id, |q| {
7763            q.abandon(why);
7764            Ok(())
7765        });
7766    }
7767
7768    /// Carry a withheld text to its end: file the question, park while it is
7769    /// open, and on an answer, silence or timeout pick the text to post. The
7770    /// decision is saved before it is used, so it applies once.
7771    async fn settle_withheld(
7772        &mut self,
7773        winner: &Candidate,
7774        pr: &PrMessage,
7775        mut g: crate::run::GithubTextAsk,
7776    ) -> Result<Option<PrMessage>> {
7777        use crate::github_text as gt;
7778        let store = ask::Questions::open();
7779        let timeout = self.state.config.graph.answer_timeout;
7780        if !g.resolved {
7781            let mut standing = g
7782                .question
7783                .as_ref()
7784                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7785            if let Some(q) = standing.as_ref()
7786                && q.status.open()
7787                && gt::expired(q, timeout)
7788            {
7789                let why = format!("no answer within {}s of asking", timeout.max(1));
7790                standing = store
7791                    .update(&q.id, |q| {
7792                        q.abandon(&why);
7793                        Ok(())
7794                    })
7795                    .ok()
7796                    .map(|(q, ())| q);
7797            }
7798            let mut ask_again: Option<Option<Vec<String>>> = None;
7799            match standing {
7800                // Recorded but never written (a stop between the two saves),
7801                // or removed: ask again without spending the retry.
7802                None if g.asks <= 1 => {
7803                    g.asks = 0;
7804                    ask_again = Some(None);
7805                }
7806                None => {
7807                    g.resolved = true;
7808                    g.chosen_title = None;
7809                    g.choice = Some(GithubTextChoice::Fallback);
7810                }
7811                Some(q) if q.status.open() => {
7812                    self.park_on_github_text(&g)?;
7813                    return Ok(None);
7814                }
7815                Some(q) => match gt::read_reply(&q) {
7816                    gt::Reply::Fallback => {
7817                        g.resolved = true;
7818                        g.chosen_title = None;
7819                        g.choice = Some(GithubTextChoice::Fallback);
7820                    }
7821                    gt::Reply::Original => {
7822                        g.resolved = true;
7823                        g.chosen_title = None;
7824                        g.choice = Some(GithubTextChoice::Original);
7825                    }
7826                    gt::Reply::Rewrite => {
7827                        g.resolved = true;
7828                        g.chosen_title = None;
7829                        g.choice = Some(if g.rewrite.is_some() {
7830                            GithubTextChoice::Rewrite
7831                        } else {
7832                            GithubTextChoice::Fallback
7833                        });
7834                    }
7835                    gt::Reply::Title(text) => {
7836                        let first = text
7837                            .lines()
7838                            .map(str::trim)
7839                            .find(|l| !l.is_empty())
7840                            .unwrap_or("");
7841                        let decision = if first.is_empty() {
7842                            None
7843                        } else {
7844                            let candidate = PrMessage {
7845                                title: first.to_owned(),
7846                                body: String::new(),
7847                            };
7848                            self.judge_pr_language(&winner.worktree, &candidate).await
7849                        };
7850                        match gt::vet_title(
7851                            &text,
7852                            decision,
7853                            self.state.config.graph.github_english(),
7854                        ) {
7855                            Ok(title) => {
7856                                g.resolved = true;
7857                                g.chosen_title = Some(title);
7858                                g.choice = Some(GithubTextChoice::MyTitle);
7859                            }
7860                            Err(cats) if g.asks < 2 => {
7861                                ask_again = Some(Some(
7862                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7863                                ))
7864                            }
7865                            Err(_) => {
7866                                g.resolved = true;
7867                                g.chosen_title = None;
7868                                g.choice = Some(GithubTextChoice::Fallback);
7869                            }
7870                        }
7871                    }
7872                },
7873            }
7874            if let Some(rejected) = ask_again {
7875                // A retry (after a rejected replacement title) is still about
7876                // the original text: the same ORIGINAL / FALLBACK comparison.
7877                let w = gt::Withheld {
7878                    title: g.title,
7879                    body: g.body,
7880                    categories: g.categories.clone(),
7881                };
7882                // Named by the original text's fingerprint alone, so a resume
7883                // rewrites the same file and the detail stays identical.
7884                let artifact = match crate::run::write_artifact(
7885                    &self.state,
7886                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7887                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7888                ) {
7889                    Ok(p) => Some(p),
7890                    Err(e) => {
7891                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7892                        None
7893                    }
7894                };
7895                let (detail, comparison) = gt::question_content(
7896                    &w,
7897                    &pr.title,
7898                    &pr.body,
7899                    &self.state.id,
7900                    rejected.as_deref(),
7901                    artifact.as_deref(),
7902                    g.rewrite.as_ref(),
7903                );
7904                let mut q = ask::Question::new(
7905                    self.state.id.clone(),
7906                    gt::ASK_NODE.to_owned(),
7907                    gt::ASK_SEAT.to_owned(),
7908                    gt::question_summary(&self.state.config.graph.language, &w),
7909                    detail,
7910                    gt::question_choices(&w, g.rewrite.as_ref()),
7911                );
7912                q.posting_comparison = Some(comparison);
7913                q.answer_timeout = timeout;
7914                // Recorded before it is written: a stop in between leaves a
7915                // record naming a question the resume then files again.
7916                g.question = Some(q.id.clone());
7917                g.asks += 1;
7918                self.state.github_text = Some(g.clone());
7919                self.state.event(
7920                    "github-text",
7921                    format!("asking the owner about the withheld text ({})", q.short()),
7922                );
7923                self.state.save()?;
7924                store
7925                    .put_executable(&mut q)
7926                    .context("file the posting-gate question")?;
7927                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7928                    tracing::warn!(
7929                        "could not notify about posting-gate question {}: {e:#}",
7930                        q.short()
7931                    );
7932                }
7933                self.park_on_github_text(&g)?;
7934                return Ok(None);
7935            }
7936        }
7937        let resolved = g.chosen_title.clone();
7938        let original = g.choice == Some(GithubTextChoice::Original);
7939        // Field names only: the event log never records the text.
7940        self.state.event(
7941            "github-text",
7942            if original {
7943                format!(
7944                    "posting the original text for: {}",
7945                    match (g.title, g.body) {
7946                        (true, true) => "title, description",
7947                        (true, false) => "title",
7948                        _ => "description",
7949                    }
7950                )
7951            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7952                "posting the saved rewrite title and description".to_owned()
7953            } else if resolved.is_some() {
7954                "posting the owner's replacement title".to_owned()
7955            } else {
7956                "posting the neutral text for the withheld fields".to_owned()
7957            },
7958        );
7959        self.state.github_text = Some(g.clone());
7960        self.state.save()?;
7961        if g.choice == Some(GithubTextChoice::Rewrite)
7962            && let Some(r) = g.rewrite.as_ref()
7963        {
7964            return Ok(Some(PrMessage {
7965                title: r.title.clone(),
7966                body: r.body.clone(),
7967            }));
7968        }
7969        let w = gt::Withheld {
7970            title: g.title,
7971            body: g.body,
7972            categories: g.categories.clone(),
7973        };
7974        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7975        if let Some(t) = resolved {
7976            title = t;
7977        }
7978        // Every withheld field is now neutral or vetted English.
7979        let english = gt::LanguageDecision {
7980            title_english: true,
7981            body_english: true,
7982            body_complete: true,
7983        };
7984        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7985        if original {
7986            // The owner's explicit choice overrides the gate for the fields it
7987            // was made for; the others keep what the gate produced.
7988            if g.title {
7989                title = pr.title.clone();
7990            }
7991            if g.body {
7992                body = pr.body.clone();
7993            }
7994        }
7995        Ok(Some(PrMessage { title, body }))
7996    }
7997
7998    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7999        self.state.github_text = Some(g.clone());
8000        self.state.end_pass_parked(ParkCause::Question);
8001        self.state.parked = true;
8002        self.state.event(
8003            "github-text",
8004            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8005        );
8006        self.state.save()
8007    }
8008
8009    /// Ask `[roles] language_judge` about this text and record which source
8010    /// decided; the text itself is never recorded.
8011    async fn judge_pr_language(
8012        &mut self,
8013        cwd: &Path,
8014        pr: &PrMessage,
8015    ) -> Option<crate::github_text::LanguageDecision> {
8016        self.state.config.roles.language_judge.as_ref()?;
8017        if !self.state.config.graph.language_gate() {
8018            return None;
8019        }
8020        let decision =
8021            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8022        self.state.event(
8023            "github-text",
8024            if decision.is_some() {
8025                "language decided by the language judge"
8026            } else {
8027                "language judge unavailable; using built-in heuristics"
8028            },
8029        );
8030        decision
8031    }
8032
8033    async fn merge(&mut self) -> Result<()> {
8034        // Same reasoning as `gate`: ask the review and gate records directly
8035        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8036        // skip can rewrite on reentry to something that no longer says
8037        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8038        // so a hand-off (open findings, green verification) reaches merge
8039        // exactly like a genuinely clean round does.
8040        //
8041        // A run resumed mid-`land` never reaches here at all: `execute`
8042        // recognises `RunStatus::Landing` before it even calls `prep`, and
8043        // routes straight to `run_land` instead. That has to happen a level
8044        // up from this function, not with a check in here, because
8045        // `review_loop`'s own status recomputation (see its doc) runs
8046        // *before* `merge` on every reentry and would otherwise overwrite
8047        // the `Landing` marker with `Gating` before this node ever saw it.
8048        if self
8049            .state
8050            .base_sync
8051            .as_ref()
8052            .is_some_and(|s| s.conflict.is_some())
8053            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8054                != Some(RunStatus::Gating)
8055            // `gate_ran == false` is not "passed" - `gate` leaves it false
8056            // both before it has ever run and when its last attempt was
8057            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8058            // permission to merge on nothing but the review record. Only a
8059            // gate that actually ran may proceed; an intentional local
8060            // review can also finish with verification explicitly not configured.
8061            // `RunState::gate_status` computes that distinction.
8062            || !self.state.gate_allows_progress()
8063        {
8064            return Ok(());
8065        }
8066        // This node's own record, not `status`: `status == Ready` is not
8067        // unique to the harmless `MergeMode::None` path this line was
8068        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8069        // run's PR was closed without merging — and on that run `mode` is
8070        // still `Pr`, so a reentry that fell through here would push and
8071        // open a second pull request. `self.state.merge` is set exactly once
8072        // this node (or `land`) has already produced a verdict, under every
8073        // mode, which is what "already done" actually means here.
8074        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8075            return Ok(());
8076        }
8077        let Some(winner) = self.state.winner().cloned() else {
8078            return Ok(());
8079        };
8080        let repo = self.state.repo.clone();
8081        let base = self.state.base_branch.clone();
8082        let mode = self.state.config.merge.mode;
8083        let style = self.state.config.merge.style;
8084        let facts = if is_review_run(&self.state) {
8085            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8086            let start = review_base(
8087                &repo,
8088                &self.state.config.merge.remote,
8089                &base,
8090                &self.state.base_commit,
8091                &winner.branch,
8092            )
8093            .await;
8094            branch_facts(&repo, &start, &winner.branch).await
8095        } else {
8096            None
8097        };
8098        // `None` when the base could not be freshly read: then an adopted
8099        // pull request's title is left alone.
8100        let leaked = if is_review_run(&self.state) {
8101            leaked_subjects(&self.state, &winner.branch).await
8102        } else {
8103            Some(Vec::new())
8104        };
8105        // `None`: parked on the owner's word about a withheld title or body.
8106        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8107        // the resume arrives here again.
8108        let Some(pr) = self
8109            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8110            .await?
8111        else {
8112            return Ok(());
8113        };
8114        let message = pr.commit_message();
8115
8116        let outcome = match mode {
8117            MergeMode::None => MergeOutcome {
8118                mode,
8119                ok: true,
8120                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8121                empty: false,
8122            },
8123            MergeMode::Pr | MergeMode::Local
8124                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8125            {
8126                MergeOutcome {
8127                    mode,
8128                    ok: false,
8129                    detail: empty_candidate_detail(&self.state, &base),
8130                    empty: true,
8131                }
8132            }
8133            MergeMode::Local => {
8134                let on = git::current_branch(&repo).await?;
8135                if on.as_deref() != Some(base.as_str()) {
8136                    MergeOutcome {
8137                        mode,
8138                        ok: false,
8139                        detail: format!(
8140                            "{} has {} checked out, not the base branch {base}",
8141                            repo.display(),
8142                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8143                        ),
8144                        empty: false,
8145                    }
8146                } else if !git::is_clean(&repo).await? {
8147                    MergeOutcome {
8148                        mode,
8149                        ok: false,
8150                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8151                        empty: false,
8152                    }
8153                } else {
8154                    let out = match style {
8155                        MergeStyle::Merge => {
8156                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8157                        }
8158                        MergeStyle::Squash => {
8159                            git::merge_squash(&repo, &winner.branch, &message).await?
8160                        }
8161                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8162                    };
8163                    MergeOutcome {
8164                        mode,
8165                        ok: out.ok(),
8166                        detail: if out.ok() { out.stdout } else { out.stderr },
8167                        empty: false,
8168                    }
8169                }
8170            }
8171            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8172                self.state.pr_publication_pending = true;
8173                self.state.save()?;
8174                let out =
8175                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8176                self.state.pr_publication_pending = out.is_err();
8177                MergeOutcome {
8178                    mode,
8179                    ok: out.is_ok(),
8180                    detail: match out {
8181                        Ok(url) => url,
8182                        Err(e) => format!("{e:#}"),
8183                    },
8184                    empty: false,
8185                }
8186            }
8187            MergeMode::Pr => {
8188                let remote = self.state.config.merge.remote.clone();
8189                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8190                if !pushed.ok() {
8191                    MergeOutcome {
8192                        mode,
8193                        ok: false,
8194                        detail: pushed.stderr,
8195                        empty: false,
8196                    }
8197                } else {
8198                    // A retry or resume of a run whose branch already has an
8199                    // open pull request adopts it rather than failing on a
8200                    // duplicate. Only this winner branch into this base:
8201                    // `branch_for` derives the name from the run id, so a
8202                    // different run's pull request never matches.
8203                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8204                    let out = match pr_merge_plan(found) {
8205                        PrPlan::Create => {
8206                            gh_pr_create(
8207                                &winner.worktree,
8208                                &base,
8209                                &winner.branch,
8210                                &pr.title,
8211                                &pr.body,
8212                            )
8213                            .await
8214                        }
8215                        PrPlan::Adopt { url, title } => {
8216                            self.state
8217                                .event("merge", format!("Pr: adopted open pull request {url}"));
8218                            if self.state.github_text.as_ref().is_some_and(|g| {
8219                                g.resolved
8220                                    && g.choice == Some(GithubTextChoice::Rewrite)
8221                                    && g.rewrite.is_some()
8222                            }) {
8223                                // The owner chose the entire pair, even if the
8224                                // existing title already matches or was edited.
8225                                land::set_pr_text_verbatim(
8226                                    &winner.worktree,
8227                                    &url,
8228                                    &pr.title,
8229                                    &pr.body,
8230                                )
8231                                .await
8232                                .map(|()| url)
8233                            } else {
8234                                if title != pr.title
8235                                    && (!is_review_run(&self.state)
8236                                        || leaked
8237                                            .as_deref()
8238                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8239                                    && let Err(e) =
8240                                        if self.state.github_text.as_ref().is_some_and(|g| {
8241                                            g.resolved
8242                                                && g.title
8243                                                && g.choice == Some(GithubTextChoice::Original)
8244                                        }) {
8245                                            // The owner chose to post this title unmodified.
8246                                            land::set_pr_title_verbatim(
8247                                                &winner.worktree,
8248                                                &url,
8249                                                &pr.title,
8250                                            )
8251                                            .await
8252                                        } else {
8253                                            land::set_pr_title(
8254                                                &mut self.state,
8255                                                &winner.worktree,
8256                                                &url,
8257                                                &pr.title,
8258                                            )
8259                                            .await
8260                                        }
8261                                {
8262                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8263                                    self.state
8264                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8265                                }
8266                                Ok(url)
8267                            }
8268                        }
8269                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8270                    };
8271                    match out {
8272                        Ok(url) => MergeOutcome {
8273                            mode,
8274                            ok: true,
8275                            detail: url,
8276                            empty: false,
8277                        },
8278                        Err(e) => MergeOutcome {
8279                            mode,
8280                            ok: false,
8281                            detail: e.to_string(),
8282                            empty: false,
8283                        },
8284                    }
8285                }
8286            }
8287        };
8288
8289        self.state.status = match (mode, outcome.ok) {
8290            (MergeMode::None, _) => RunStatus::Ready,
8291            (_, true) if outcome.empty => RunStatus::Ready,
8292            (MergeMode::Pr, true) => RunStatus::Ready,
8293            (_, true) => RunStatus::Merged,
8294            (_, false) => RunStatus::Blocked,
8295        };
8296        self.state.event(
8297            "merge",
8298            format!(
8299                "{:?}: {}",
8300                mode,
8301                outcome.detail.lines().next().unwrap_or("")
8302            ),
8303        );
8304        self.state.merge = Some(outcome);
8305        self.state.save()?;
8306
8307        // The PR is open and the run would historically stop here, leaving the
8308        // operator to watch checks, feed review comments back to a fixer, and
8309        // merge. That was done by hand six times in one session before this
8310        // existed. Opt-in, because merging is the one irreversible thing magi
8311        // can do to a repository.
8312        if self.state.config.graph.land
8313            && mode == MergeMode::Pr
8314            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8315        {
8316            self.run_land().await?;
8317        }
8318        // `run_land` may have left `status` at `Landing` - still waiting on
8319        // CI or the owner's approval, not actually settled - so this has to
8320        // read whatever `status` ended up as here, not the `Merged` this
8321        // function set a few lines up.
8322        self.settle_questions();
8323        Ok(())
8324    }
8325
8326    /// Enter `land`.
8327    ///
8328    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8329    /// a resumed run's re-entry. `land::land` itself is what serialises the
8330    /// two git-mutating moments inside the loop — the rebase push and
8331    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8332    /// to hold a lock across the whole call, and doing so would serialise
8333    /// this run's CI wait against a *different* run's land-approval resume
8334    /// in the same repository, which is exactly the "must not wait on
8335    /// another task" property the daemon's slot-freeing exists to give.
8336    async fn run_land(&mut self) -> Result<()> {
8337        crate::pr_head::ensure_owner(&self.state)?;
8338        let url = self
8339            .state
8340            .pr
8341            .as_ref()
8342            .map(|pr| pr.url.clone())
8343            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8344            .unwrap_or_default();
8345        let url = url.lines().next().unwrap_or("").trim().to_owned();
8346        if !url.starts_with("http") {
8347            return Ok(());
8348        }
8349        // A land failure is not a lost run: the work is on a branch and the
8350        // pull request is open, which is exactly where a human takes over.
8351        loop {
8352            match land::land(&mut self.state, &url).await {
8353                Ok(_)
8354                    if self
8355                        .state
8356                        .merge_repairs
8357                        .last()
8358                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8359                {
8360                    if self.repair_merge_review().await? {
8361                        continue;
8362                    }
8363                }
8364                Ok(pr) if self.state.parked => {
8365                    // `land` already saved the parked marker; nothing here
8366                    // overrides `status` back to a terminal value while an
8367                    // approval is still outstanding.
8368                    let _ = pr;
8369                }
8370                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8371                    self.state.status = RunStatus::Superseded;
8372                    self.state.save()?;
8373                }
8374                Ok(pr) => {
8375                    self.state.status = match pr.state {
8376                        land::PrLifecycle::Merged => RunStatus::Merged,
8377                        _ => RunStatus::Blocked,
8378                    };
8379                    // Downstream of a confirmed merge only - see
8380                    // `bump::should_release_bump`'s own doc for why this one
8381                    // check covers all three of `land`'s success paths.
8382                    // Best-effort: the run already landed, so a failure here
8383                    // (the decision call, `gh`, `cargo`) is recorded and never
8384                    // turns a landed run into a failed one.
8385                    if bump::should_release_bump(self.state.status)
8386                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8387                    {
8388                        // The event is the run's own record. Not-eligible cases
8389                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8390                        // `Err` is a bump that was tried and failed:
8391                        // `after_merge` itself raises the operator notice for
8392                        // that, whether or not a release PR exists yet.
8393                        self.state
8394                            .event("bump", format!("release bump skipped: {e:#}"));
8395                    }
8396                    // Independent of the bump, and best-effort in the same way:
8397                    // findings the merge left open become follow-up tasks.
8398                    if self.state.status == RunStatus::Merged {
8399                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8400                    }
8401                    self.state.save()?;
8402                }
8403                Err(e) => {
8404                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8405                        return Err(e);
8406                    }
8407                    self.state.status = RunStatus::Blocked;
8408                    self.state.event("land", format!("gave up: {e}"));
8409                    self.state.save()?;
8410                }
8411            }
8412            break;
8413        }
8414        Ok(())
8415    }
8416
8417    /// One repair cycle stays under the existing driver/task claim. An
8418    /// interrupted fixer is rolled back and held rather than run twice.
8419    async fn repair_merge_review(&mut self) -> Result<bool> {
8420        use crate::run::MergeRepairPhase as Phase;
8421        crate::pr_head::ensure_owner(&self.state)?;
8422        let i = self.state.merge_repairs.len() - 1;
8423        let request = self.state.merge_repairs[i].clone();
8424        if request.phase == Phase::Fixing {
8425            let restored = match self.state.winner().cloned() {
8426                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8427                None => Err(anyhow::anyhow!("repair has no winning branch")),
8428            };
8429            let why = match restored {
8430                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8431                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8432                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8433            };
8434            self.hold_merge_repair(i, why)?;
8435            return Ok(false);
8436        }
8437        let failure = if i >= self.state.config.graph.land_rounds
8438            || self.state.config.graph.review_rounds == 0
8439        {
8440            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8441        } else {
8442            None
8443        };
8444        if let Some(why) = failure {
8445            self.hold_merge_repair(i, why)?;
8446            return Ok(false);
8447        }
8448        if request.phase == Phase::Requested {
8449            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8450                Ok(pr) => pr,
8451                Err(e) => {
8452                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8453                    return Ok(false);
8454                }
8455            };
8456            self.state.merge_repairs[i].phase = Phase::Fixing;
8457            self.state.land_approval = None;
8458            self.state.land_armed_head = None;
8459            self.state.save()?;
8460            // Serialize the exact records, including finding ids/details,
8461            // reconsideration reasons, final votes and verification output.
8462            let evidence = serde_json::to_string_pretty(&request)?;
8463            let reason = format!(
8464                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8465            );
8466            let budget = self.state.config.graph.land_rounds;
8467            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8468            match result {
8469                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8470                Ok(land::Fixed::Failed(why)) => {
8471                    self.hold_merge_repair(
8472                        i,
8473                        format!(
8474                            "repair failed: {why}; inspect the branch and run an explicit review"
8475                        ),
8476                    )?;
8477                    return Ok(false);
8478                }
8479                Err(e) => {
8480                    self.hold_merge_repair(
8481                        i,
8482                        format!(
8483                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8484                        ),
8485                    )?;
8486                    return Ok(false);
8487                }
8488            }
8489            // Even no change gets a real new reviewer wave. The previous
8490            // evidence remains in the request; a former clean/concluded round
8491            // cannot short-circuit the new review or gate.
8492            self.begin_merge_repair_review(i)?;
8493        }
8494        self.review_loop().await?;
8495        self.gate().await?;
8496        if self.resource_halt()? || self.state.parked {
8497            return Ok(false);
8498        }
8499        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8500            != Some(RunStatus::Gating)
8501            || !self.state.gate_allows_progress()
8502        {
8503            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8504            return Ok(false);
8505        }
8506        // Review and gate may have added commits after the initial repair.
8507        // Publish exactly their final head to the existing PR, with a lease.
8508        let winner = self
8509            .state
8510            .winner()
8511            .cloned()
8512            .context("repair has no winning branch")?;
8513        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8514        if self.state.gate_head.as_deref() != Some(after.as_str())
8515            || !git::is_clean(&winner.worktree).await?
8516        {
8517            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8518            return Ok(false);
8519        }
8520        let expected = self
8521            .state
8522            .pr_pushed_head
8523            .as_deref()
8524            .unwrap_or(&request.head)
8525            .to_owned();
8526        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8527        // Push the checked commit itself: a local ref moving after the check
8528        // must not change which commit the forge receives.
8529        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8530        let refspec = format!("{after}:refs/heads/{public}");
8531        let push = git::git_raw(
8532            &winner.worktree,
8533            &["push", &lease, &self.state.config.merge.remote, &refspec],
8534        )
8535        .await;
8536        match push {
8537            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8538            Ok(out) => {
8539                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8540                return Ok(false);
8541            }
8542            Err(e) => {
8543                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8544                return Ok(false);
8545            }
8546        }
8547        self.state.merge_repairs[i].phase = Phase::Complete;
8548        self.state.status = RunStatus::Landing;
8549        self.state.event(
8550            "repair",
8551            "new review and gate completed; fresh owner merge decision required",
8552        );
8553        self.state.save()?;
8554        Ok(true)
8555    }
8556
8557    fn merge_repair_artifacts(&self) -> PathBuf {
8558        let artifacts = agent::artifacts_dir(&self.state.dir());
8559        match self
8560            .state
8561            .merge_repairs
8562            .last()
8563            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8564        {
8565            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8566            None => artifacts,
8567        }
8568    }
8569
8570    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8571        self.state.reviews.clear();
8572        self.state.gate.clear();
8573        self.state.gate_ran = false;
8574        self.state.gate_head = None;
8575        self.state.gate_fixes.clear();
8576        self.state.gate_repair_e2e = None;
8577        self.state.pre_gate.clear();
8578        self.state.pre_gate_commit = None;
8579        self.state.contested_handoff = None;
8580        self.state.land_reruns = None;
8581        self.state.land_approval = None;
8582        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8583        self.state.status = RunStatus::Reviewing;
8584        self.state.save()
8585    }
8586
8587    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8588        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8589        self.state.merge_repairs[index].failure = Some(why.clone());
8590        self.state.status = RunStatus::Blocked;
8591        self.state.event("repair", why);
8592        self.state.save()
8593    }
8594
8595    // -------------------------------------------------------------- helpers
8596
8597    /// Fetch or create a seat, keeping its conversation across nodes.
8598    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8599        if let Some(existing) = self.state.seats.get(key)
8600            && existing.agent == agent
8601        {
8602            return existing.clone();
8603        }
8604        // A seat that changes agent mints its session id from the agent too,
8605        // like a handover: the old agent's uuid is already taken by the CLI.
8606        let fresh = if self.state.seats.contains_key(key) {
8607            handover_seat(key, agent, self.state.next_seat_seed())
8608        } else {
8609            SeatState::new(key, agent, self.state.seed)
8610        };
8611        self.state.seats.insert(key.to_owned(), fresh.clone());
8612        fresh
8613    }
8614
8615    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8616    /// seat to another roster agent, in which case that agent. Nodes that
8617    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8618    /// reconsideration) must keep talking to whoever answered it, not slip
8619    /// back to the agent that failed it.
8620    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8621        match self.state.seats.get(key) {
8622            Some(s) if s.agent != spec.id => {
8623                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8624            }
8625            _ => spec,
8626        }
8627    }
8628
8629    /// A candidate rendered for judging, with the leak policy applied.
8630    fn view(&self, c: &Candidate) -> CandidateView {
8631        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8632            .unwrap_or_default();
8633        let (patch, _) = blind::sanitize_patch(
8634            &format!("candidate {} patch", c.label),
8635            &raw,
8636            &self.state.config.blind,
8637        );
8638        CandidateView {
8639            label: c.label,
8640            branch: c.branch.clone(),
8641            summary: c.summary.clone(),
8642            stat: c.stat.clone(),
8643            patch,
8644        }
8645    }
8646
8647    /// The full candidate set as prompt text, for seats with no live session.
8648    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8649        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8650        prompt::judge(
8651            "(see above)",
8652            &views,
8653            self.roles.judges.len(),
8654            base_short,
8655            "en",
8656        )
8657    }
8658
8659    /// The final-vote prompt with everything a seat that has no session of its
8660    /// own needs: the candidates, the seat's own ranking and reasons, and the
8661    /// anonymised deliberation it took part in (only when there was one, so a
8662    /// handed-over seat never sees more than the seat it replaces did). The
8663    /// `Final vote` heading stays first.
8664    fn vote_prompt_full(
8665        &self,
8666        j: usize,
8667        labels: &[char],
8668        language: &str,
8669        candidates: &[Candidate],
8670        base_short: &str,
8671    ) -> String {
8672        let mut text = format!(
8673            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8674            prompt::final_vote(labels, language),
8675            self.state.instruction,
8676            self.candidate_block(candidates, base_short)
8677        );
8678        if let Some(own) = self
8679            .state
8680            .judgements
8681            .get(j)
8682            .filter(|r| !r.ranking.is_empty())
8683        {
8684            let reasons = own
8685                .reasons
8686                .iter()
8687                .map(|(k, v)| format!("- {k}: {v}"))
8688                .collect::<Vec<_>>()
8689                .join("\n");
8690            text.push_str(&format!(
8691                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8692                own.ranking.iter().collect::<String>(),
8693                if reasons.is_empty() {
8694                    ""
8695                } else {
8696                    ", because:\n"
8697                }
8698            ));
8699        }
8700        if !self.state.deliberation.is_empty() {
8701            text.push_str("\n# What was argued before this vote\n");
8702            for t in self.transcript(&[], j) {
8703                text.push_str(&format!(
8704                    "\n## {}{}\n\n{}\n",
8705                    t.who,
8706                    if t.is_self { " (you)" } else { "" },
8707                    t.body.trim()
8708                ));
8709            }
8710        }
8711        text
8712    }
8713
8714    /// Anonymised transcript for judge `self_idx`.
8715    ///
8716    /// The initial rankings are always the opening statements. Seeding them
8717    /// only when no turn had been taken yet meant every judge after the first
8718    /// argued against a single voice instead of against the actual split — the
8719    /// disagreement is the information, so it is always on the table.
8720    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8721        let mut turns = Vec::new();
8722        for j in &self.state.judgements {
8723            if j.ranking.is_empty() {
8724                continue;
8725            }
8726            let reasons = j
8727                .reasons
8728                .iter()
8729                .map(|(k, v)| format!("- {k}: {v}"))
8730                .collect::<Vec<_>>()
8731                .join("\n");
8732            turns.push(Turn {
8733                who: format!("Judge {} (opening ranking)", j.judge),
8734                is_self: j.judge == self_idx + 1,
8735                body: format!(
8736                    "Ranked {}{}{reasons}",
8737                    j.ranking.iter().collect::<String>(),
8738                    if reasons.is_empty() {
8739                        ""
8740                    } else {
8741                        ", because:\n"
8742                    }
8743                ),
8744            });
8745        }
8746        for t in self
8747            .state
8748            .deliberation
8749            .iter()
8750            .flat_map(|r| r.turns.iter())
8751            .chain(current)
8752        {
8753            turns.push(Turn {
8754                who: format!("Judge {}", t.judge),
8755                is_self: t.judge == self_idx + 1,
8756                body: t.body.clone(),
8757            });
8758        }
8759        turns
8760    }
8761}
8762
8763/// Does this seat still hold the context a follow-up prompt would rely on?
8764fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8765    agent::has_session(spec.kind, seat, sessions)
8766}
8767
8768/// The next entry in `roster` after `start`, never wrapping back to the
8769/// front, whose id is not in `tried` yet.
8770///
8771/// Starts one past `start` rather than at the front of `roster`: `start` is
8772/// the seat's own original position, and a seat whose candidate slot already
8773/// sits on the roster's second entry must fall through to the third next, not
8774/// restart at the first — which is very likely a different candidate's own
8775/// agent already. Never wraps back past `start`, for the same reason: an
8776/// entry earlier in the roster than the seat's own position is almost
8777/// certainly some *other* candidate slot's own agent, and once the tail of
8778/// the roster is exhausted there are no more untried agents for *this* seat
8779/// to fall through to — the caller's fallback chain ends there, exactly as
8780/// "no further untried agents remain in the list for that seat" asks for.
8781///
8782/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8783/// the same id twice (an operator's `roles.implementers` typo, or a
8784/// `[[agents]]` list reused across roles) must not let
8785/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8786/// over `roster` either finds an untried id or runs out, so this always
8787/// terminates regardless of duplicates.
8788fn next_untried_in_roster<'a>(
8789    roster: &'a [AgentSpec],
8790    start: usize,
8791    tried: &BTreeSet<String>,
8792) -> Option<&'a AgentSpec> {
8793    roster
8794        .get(start + 1..)?
8795        .iter()
8796        .find(|s| !tried.contains(&s.id))
8797}
8798
8799/// The successor for a seat, shared by all four seat kinds (implement, judge,
8800/// review, advise). `others` holds the ids that *currently* occupy the other
8801/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8802/// sees them), so roster entries beyond the seat count act as spares: a failed
8803/// seat goes to an agent no other seat holds whenever the roster permits.
8804///
8805/// `carried` is `Some` only for the review loop's carried failure history.
8806/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8807/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8808/// whole roster: untried and not another seat's occupant; (3) the plain walk
8809/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8810/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8811/// a better outcome than an empty seat, and it keeps the answer to "is there
8812/// a successor at all" exactly what it was before occupants were considered,
8813/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8814/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8815/// carried failure is a cheaper bet than doubling an agent on the panel, and
8816/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8817/// the same round are handled one at a time, so a seat later in the batch
8818/// sees an earlier one's new occupant but not yet its own freed agent.
8819fn pick_successor<'a>(
8820    roster: &'a [AgentSpec],
8821    start: usize,
8822    tried: &BTreeSet<String>,
8823    carried: Option<&BTreeSet<String>>,
8824    others: &BTreeSet<String>,
8825) -> Option<&'a AgentSpec> {
8826    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8827    roster
8828        .get(start + 1..)
8829        .and_then(|tail| {
8830            tail.iter()
8831                .filter(free)
8832                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8833        })
8834        .or_else(|| {
8835            carried
8836                .is_some()
8837                .then(|| roster.iter().find(free))
8838                .flatten()
8839        })
8840        .or_else(|| match carried {
8841            Some(c) => next_for_seat(roster, start, tried, c),
8842            None => next_untried_in_roster(roster, start, tried),
8843        })
8844}
8845
8846/// The next agent for a seat that carries its failure history across rounds
8847/// (the review loop). `round_tried` is this round's own bound and starts
8848/// empty every round; `carried_failed` only decides priority.
8849///
8850/// First: an id walking forward from `start`, never wrapping, that is neither
8851/// tried this round nor failed in an earlier one. Only when that is exhausted
8852/// does it rescue: the first roster id (in roster order, so this one *does*
8853/// look before `start`) not yet tried this round, which is by then a carried
8854/// failure. Each id is rescued at most once per round, so it cannot loop.
8855fn next_for_seat<'a>(
8856    roster: &'a [AgentSpec],
8857    start: usize,
8858    round_tried: &BTreeSet<String>,
8859    carried_failed: &BTreeSet<String>,
8860) -> Option<&'a AgentSpec> {
8861    roster
8862        .get(start + 1..)?
8863        .iter()
8864        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8865        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8866}
8867
8868/// Where a reviewer seat starts a round: the agent that last answered it when
8869/// it is still on the roster and not marked failed, else the spec's own agent
8870/// unless it failed, else the next roster agent that has not failed, else the
8871/// spec's own agent again (the whole roster failed). Ids no longer on the
8872/// roster are ignored. An empty roster has no handover, so the spec stands.
8873fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8874    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8875        return spec;
8876    };
8877    let ok = |id: &str| !h.failed.contains(id);
8878    if let Some(last) = h.last_ok.as_deref()
8879        && ok(last)
8880        && let Some(s) = roster.iter().find(|s| s.id == last)
8881    {
8882        return s.clone();
8883    }
8884    if ok(&spec.id) {
8885        return spec;
8886    }
8887    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8888    roster
8889        .get(from + 1..)
8890        .into_iter()
8891        .flatten()
8892        .chain(roster.iter())
8893        .find(|s| ok(&s.id))
8894        .cloned()
8895        .unwrap_or(spec)
8896}
8897
8898/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
8899/// seed so a CLI that mints its session id up front (`--session-id`) never
8900/// reuses the uuid the previous agent already opened under the same seat key.
8901pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8902    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8903}
8904
8905/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
8906/// so the classifier below and the code that builds the message cannot drift.
8907const TIMED_OUT: &str = "timed out";
8908
8909impl FailClass {
8910    /// `None` for an answer; otherwise how the turn failed.
8911    fn of(out: &AgentOutcome) -> Option<Self> {
8912        match out {
8913            AgentOutcome::Ok(_) => None,
8914            AgentOutcome::Quota(_) => Some(Self::Quota),
8915            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8916            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8917            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8918            // Not a failure of the agent. Callers check for it first; `None`
8919            // here means "no class to hand over on".
8920            AgentOutcome::ResourceBlocked(_) => None,
8921        }
8922    }
8923
8924    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
8925    fn stem_word(&self) -> &'static str {
8926        match self {
8927            Self::Quota => "quota",
8928            _ => "handover",
8929        }
8930    }
8931}
8932
8933/// The message's first line with its variable parts removed — digit runs and
8934/// path-like tokens — so "exit code 2" and "exit code 7" read
8935/// as one kind of failure.
8936fn failure_signature(msg: &str) -> String {
8937    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8938    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
8939    // the historical machine key while the process diagnosis uses readable prose.
8940    if let Some(code) = line
8941        .strip_prefix("process exited with no usable output (exit code ")
8942        .and_then(|s| s.strip_suffix(')'))
8943        .and_then(|s| s.parse::<i32>().ok())
8944    {
8945        line = format!("exited with some({code}) and no usable output");
8946    } else if matches!(
8947        line.as_str(),
8948        "process exited with no usable output (no exit code reported)"
8949            | "process exited with no usable output (exit code unavailable)"
8950    ) {
8951        line = "exited with none and no usable output".to_owned();
8952    }
8953    let mut out = Vec::new();
8954    for word in line.split_whitespace() {
8955        if word.contains('/') || word.contains('\\') {
8956            out.push("<path>".to_owned());
8957            continue;
8958        }
8959        let mut w = String::new();
8960        let mut in_digits = false;
8961        for c in word.chars() {
8962            if c.is_ascii_digit() {
8963                if !in_digits {
8964                    w.push('#');
8965                }
8966                in_digits = true;
8967            } else {
8968                in_digits = false;
8969                w.push(c);
8970            }
8971        }
8972        out.push(w);
8973    }
8974    out.join(" ").chars().take(120).collect()
8975}
8976
8977/// Whether a seat that just failed with `cur` may go to the next roster agent.
8978/// A quota or a timeout always may. Any other failure may not when the agent
8979/// before it failed the same way: an error the prompt causes would otherwise
8980/// walk the whole roster. `prev` is the class of the immediately preceding
8981/// agent's failure, so a quota or timeout in between breaks the run of
8982/// identical failures by itself.
8983fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8984    match cur {
8985        FailClass::Quota | FailClass::Timeout => true,
8986        FailClass::Other(_) => prev != Some(cur),
8987    }
8988}
8989
8990/// A short human reason for a failed outcome, for the handover record.
8991fn fail_reason(out: &AgentOutcome) -> String {
8992    match out {
8993        AgentOutcome::Ok(_) => String::new(),
8994        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8995        AgentOutcome::Dropped(o) => format!(
8996            "the CLI dropped the stream ({})",
8997            o.dropped
8998                .as_ref()
8999                .map(|d| d.why.as_str())
9000                .unwrap_or("it ended without delivering its answer")
9001        ),
9002        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9003            e.lines().next().unwrap_or("").chars().take(160).collect()
9004        }
9005    }
9006}
9007
9008/// Note one handover in the run: the structured record and, in the timeline,
9009/// the sentence a person reads. A quota keeps the wording it always had.
9010pub(crate) fn record_handover(
9011    state: &mut RunState,
9012    node: &str,
9013    seat: &str,
9014    from: &str,
9015    to: &str,
9016    class: &FailClass,
9017    reason: &str,
9018) {
9019    let message = if *class == FailClass::Quota {
9020        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9021    } else {
9022        format!("{seat}: handed over {from} -> {to} ({reason})")
9023    };
9024    state.event(node, message);
9025    state.handovers.push(Handover {
9026        at: Timestamp::now(),
9027        node: node.to_owned(),
9028        seat: seat.to_owned(),
9029        from: from.to_owned(),
9030        to: to.to_owned(),
9031        reason: reason.to_owned(),
9032    });
9033}
9034
9035/// Did this reply report running a command whose own CLI never confirmed an
9036/// exit status?
9037///
9038/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9039/// command *finished* (see that type's own doc), so this can only be `true`
9040/// for a command whose completion event carried no readable exit code — not
9041/// for one that simply is not mentioned at all. That is the one signal this
9042/// crate can read, from the same record `state.jobs` renders, about a reply
9043/// standing next to work its own CLI cannot vouch for finishing; it is
9044/// deliberately not a check on the exit code's *value* (a fixer legitimately
9045/// runs a command that fails mid-iteration before it succeeds) and not a
9046/// guess at a command still running in the background (which emits no event
9047/// at all, and so leaves no evidence here to find).
9048fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9049    commands.iter().any(|c| c.exit_code.is_none())
9050}
9051
9052/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9053/// trusted as a verified no-op — the adoption guard's own text-level half.
9054///
9055/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9056/// timed out): a marker only earns the benefit of the doubt from a turn the
9057/// CLI itself vouches for finishing properly, the same house style
9058/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9059/// *fix* report to for `commands`. A candidate that timed out, exited
9060/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9061/// is, whatever prose it wrote — this returns `None` before it ever looks at
9062/// `text`. The remaining guards (the tree really is empty, the evidence is
9063/// non-empty) are the caller's: this only reads what the reply *claimed*.
9064fn verified_noop_claim(
9065    usable: bool,
9066    commands: &[agent::CommandEvidence],
9067    text: &str,
9068) -> Option<String> {
9069    (usable && !has_unconfirmed_command(commands))
9070        .then(|| verdict::verified_noop(text))
9071        .flatten()
9072}
9073
9074fn short(commit: &str) -> String {
9075    commit.chars().take(7).collect()
9076}
9077
9078fn make_executable(path: &Path) -> Result<()> {
9079    #[cfg(unix)]
9080    {
9081        use std::os::unix::fs::PermissionsExt as _;
9082        let mut perms = std::fs::metadata(path)?.permissions();
9083        perms.set_mode(0o755);
9084        std::fs::set_permissions(path, perms)?;
9085    }
9086    #[cfg(not(unix))]
9087    {
9088        let _ = path;
9089    }
9090    Ok(())
9091}
9092
9093/// What every seat in one batch shares: where the answers are attributed, the
9094/// prompt overlay they inherit, and the build cache they are told to use.
9095///
9096/// A struct rather than four more parameters: `wave` also needs the run's
9097/// state (to record who is answering right now) and the attempt number, and
9098/// eight positional arguments is both unreadable and a clippy error.
9099struct WaveCtx<'a> {
9100    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9101    /// paid for it.
9102    run: &'a str,
9103    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9104    node: &'a str,
9105    prompts: &'a Prompts,
9106    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9107    cache: Option<&'a Path>,
9108    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9109    /// `JobRecord::round`. `None` for every other node.
9110    round: Option<usize>,
9111    /// Carry each seat's failed-agent history across waves (the review loop
9112    /// only): start-of-round priority and handover choice read
9113    /// [`RunState::seat_history`], and every answer or failure writes it.
9114    carry_seats: bool,
9115    /// The run's park / interrupt requests, so a wait for a shared resource
9116    /// can be abandoned the moment the operator asks to park.
9117    pauses: Option<(&'a Pause, &'a Pause)>,
9118}
9119
9120/// Shared scheduling policy for operations and spawned seat invocations.
9121fn repository_wait(
9122    state: &RunState,
9123    pauses: Option<(&Pause, &Pause)>,
9124) -> crate::repo_lock::WaitPolicy {
9125    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9126    let inherited = crate::repo_lock::WaitPolicy::current();
9127    crate::repo_lock::WaitPolicy::new(
9128        Duration::from_secs(state.config.graph.timeout_resource_wait),
9129        move || {
9130            inherited.cancelled()
9131                || pauses
9132                    .as_ref()
9133                    .is_some_and(|(a, b)| a.parked() || b.parked())
9134        },
9135    )
9136}
9137
9138fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9139    if error.is::<crate::repo_lock::ResourceBlocked>() {
9140        state.resource_block = Some(crate::run::ResourceBlock {
9141            node: "repository".into(),
9142            kind: if error.to_string().contains("disk pressure") {
9143                "disk pressure"
9144            } else {
9145                "repository coordination"
9146            }
9147            .into(),
9148            detail: error.to_string(),
9149            at: Timestamp::now(),
9150        });
9151        state.event("resource", error.to_string());
9152    }
9153}
9154
9155async fn coordinate_operation(
9156    state: &mut RunState,
9157    cwd: &Path,
9158    wait: &crate::repo_lock::WaitPolicy,
9159) -> Result<Option<crate::repo_lock::SeatLease>> {
9160    let result =
9161        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9162            .await;
9163    if let Err(e) = &result {
9164        record_repository_block(state, e);
9165    }
9166    result
9167}
9168
9169/// Run one job, honouring the parallelism budget.
9170async fn run_one(
9171    job: SeatJob,
9172    sem: Arc<Semaphore>,
9173    ctx: &WaveCtx<'_>,
9174    state: &mut RunState,
9175    attempt: usize,
9176) -> (SeatState, AgentOutcome) {
9177    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9178        .await
9179        .pop()
9180        .expect("one job in, one result out");
9181    (seat, out)
9182}
9183
9184/// Run every job concurrently, capped by the semaphore, preserving order.
9185///
9186/// Every seat in the batch is recorded into [`RunState::active`] before the
9187/// wave starts and cleared as each answer lands, so the run's own record says
9188/// who is still being waited on rather than only who finished.
9189async fn wave(
9190    jobs: Vec<SeatJob>,
9191    sem: Arc<Semaphore>,
9192    ctx: &WaveCtx<'_>,
9193    state: &mut RunState,
9194    attempt: usize,
9195) -> Vec<(usize, SeatState, AgentOutcome)> {
9196    let WaveCtx {
9197        run,
9198        node,
9199        prompts,
9200        cache,
9201        round,
9202        carry_seats: _,
9203        pauses,
9204    } = *ctx;
9205    for job in &jobs {
9206        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9207    }
9208    if let Err(e) = state.save() {
9209        // A failed persist of "who is answering right now" must not abort the
9210        // wave: the seats are already being asked, and the alternative is
9211        // losing the answers to save a status line nobody may even be
9212        // watching.
9213        tracing::warn!("could not persist in-progress seats: {e:#}");
9214    }
9215    // Hold the shared build cache's lease for the whole batch, not per job:
9216    // several candidates (an implement wave) or a fixer legitimately share
9217    // one cache concurrently within this run, and that stays untouched — a
9218    // single lease taken once for the whole wave and released once it is
9219    // done is what stops a *different* borrower (another run's own wave, its
9220    // e2e/gate, a human's `magi review`) from interleaving a build into the
9221    // same directory while this one is in flight. Best-effort, not
9222    // all-or-nothing: a wave that cannot get the lease within its own
9223    // longest job's budget still runs — an hour of paid implementer calls is
9224    // not thrown away over cache contention — but every write-allowed seat
9225    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9226    // below), the same fallback a read-only seat always gets, rather than
9227    // building into a directory this run was never granted. The identity
9228    // record is still invalidated below either way, so the next tracked
9229    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9230    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9231    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9232    // never borrowed from a job's agent budget (and never charged to it either,
9233    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9234    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9235    let cache_guard = if let Some(cache_dir) = cache {
9236        if jobs_had_a_writer {
9237            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9238            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9239            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9240            {
9241                Ok(Some(g)) => Some(g),
9242                // Parking: leave without spawning anything. The run parks at
9243                // the next boundary and the seats are asked afresh on resume.
9244                Ok(None) => {
9245                    state.resource_block = Some(crate::run::ResourceBlock {
9246                        node: node.to_owned(),
9247                        kind: "shared build cache".to_owned(),
9248                        detail: "the run is parking".to_owned(),
9249                        at: Timestamp::now(),
9250                    });
9251                    return block_wave(
9252                        jobs,
9253                        state,
9254                        node,
9255                        attempt,
9256                        "parked while waiting for the shared build cache",
9257                    );
9258                }
9259                // An unreadable lease is not contention: keep the old
9260                // fallback (no shared cache for this wave's seats).
9261                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9262                Err(e) => {
9263                    // Writing seats may not build into a cache this run does
9264                    // not hold, and an agent that cannot be given one is not
9265                    // asked at all: the run stops resumably instead.
9266                    let why = format!("{e:#}");
9267                    state.resource_block = Some(crate::run::ResourceBlock {
9268                        node: node.to_owned(),
9269                        kind: "shared build cache".to_owned(),
9270                        detail: why.clone(),
9271                        at: Timestamp::now(),
9272                    });
9273                    state.event(
9274                        "resource",
9275                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9276                    );
9277                    return block_wave(jobs, state, node, attempt, &why);
9278                }
9279            }
9280        } else {
9281            None
9282        }
9283    } else {
9284        None
9285    };
9286    // Every job keeps its whole budget: the wait above was not the agent's.
9287    // Restart the seats' clocks so "started" means "being spawned now".
9288    state.seats_spawning(node, attempt);
9289    if let Err(e) = state.save() {
9290        tracing::warn!("could not persist the start of a wave: {e:#}");
9291    }
9292    // Queueing behind this wave's own seats for a parallelism slot is normal
9293    // (each of them is bounded by its own agent timeout); only a wait beyond
9294    // all of them plus the resource limit means the slots never came back.
9295    let slot_limit = resource_limit
9296        + jobs
9297            .iter()
9298            .map(|j| j.timeout)
9299            .max()
9300            .unwrap_or_default()
9301            .saturating_mul(jobs.len() as u32);
9302    let mut set = tokio::task::JoinSet::new();
9303    // Seats announce themselves here the moment they hold a parallelism
9304    // permit, i.e. just before their CLI is spawned.
9305    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9306    let overlay = prompts.overlay(node);
9307    let repository_wait = repository_wait(state, pauses);
9308    for (i, mut job) in jobs.into_iter().enumerate() {
9309        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9310        if cache.is_some() {
9311            job.prompt.push('\n');
9312            job.prompt
9313                .push_str(&prompt::build_cache_note(node, job.allow_write));
9314        }
9315        let sem = Arc::clone(&sem);
9316        let run = run.to_owned();
9317        let node = node.to_owned();
9318        // Only implementers were told about the task's attachments, so only
9319        // their seats get the directory widened for reading.
9320        let attachments = if node == "implement" {
9321            state.attachments.clone()
9322        } else {
9323            Vec::new()
9324        };
9325        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9326        // `prompt::build_cache_note`'s doc for why setting it anyway is
9327        // exactly how a sandboxed reviewer's write refusal got reported as a
9328        // defect in the patch, not a property of its own seat. And a
9329        // write-allowed one is handed it only when the lease above was
9330        // actually acquired: a wave that could not get it (`cache_guard` is
9331        // `None`, see its own comment) must not send seats to build into a
9332        // directory this run does not hold - that is the exact concurrent,
9333        // unmanaged-write race this module exists to prevent, not something
9334        // "proceeding anyway" is allowed to reintroduce.
9335        let cache = cache
9336            .filter(|_| job.allow_write && cache_guard.is_some())
9337            .map(Path::to_path_buf);
9338        let spawned_tx = spawned_tx.clone();
9339        let repository_wait = repository_wait.clone();
9340        let coordination = state.config.coordination.clone();
9341        let fetched_base = state.fetched_base.clone();
9342        set.spawn(async move {
9343            // A wait for a parallelism slot is scheduling, not the agent: it
9344            // is bounded by its own limit and ends as a resource block, never
9345            // as a timeout of the agent that was not started.
9346            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9347                Ok(p) => p,
9348                Err(_) => {
9349                    return (
9350                        i,
9351                        job.seat,
9352                        AgentOutcome::ResourceBlocked(format!(
9353                            "no parallel slot became free within {}s",
9354                            slot_limit.as_secs()
9355                        )),
9356                        false,
9357                    );
9358                }
9359            };
9360            spawned_tx.send(job.seat.key.clone()).ok();
9361            let mut seat = job.seat;
9362            let out = repository_wait
9363                .scope(agent::invoke(
9364                    &job.spec,
9365                    &mut seat,
9366                    &Invocation {
9367                        scope: crate::agent::InvocationScope::Repository,
9368                        coordination: coordination.as_ref(),
9369                        fetched_base: fetched_base.as_ref(),
9370                        cwd: &job.cwd,
9371                        prompt: &job.prompt,
9372                        timeout: job.timeout,
9373                        allow_write: job.allow_write,
9374                        unsandboxed: false,
9375                        sessions: job.sessions,
9376                        artifacts: &job.artifacts,
9377                        stem: &job.stem,
9378                        run: &run,
9379                        node: &node,
9380                        cache_dir: cache.as_deref(),
9381                        attachments: &attachments,
9382                        writable: &[],
9383                    },
9384                ))
9385                .await;
9386            let interrupted = out
9387                .as_ref()
9388                .err()
9389                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9390            let out = match out {
9391                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9392                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9393                // Billed work the CLI failed to hand over is not an ordinary
9394                // failure, but its text is the CLI's raw error JSON, not an
9395                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9396                // read it as one by forgetting to check. `usable()` is always
9397                // false here (dropped implies an empty response), so this has
9398                // to be checked before the catch-all `Failed` below or the
9399                // one shape this exists for is lost with the rest.
9400                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9401                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9402                Ok(o) => AgentOutcome::Failed(format!(
9403                    "process exited with no usable output ({})",
9404                    crate::notices::exit_status(o.exit_code)
9405                )),
9406                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9407                    AgentOutcome::ResourceBlocked(e.to_string())
9408                }
9409                Err(e) => AgentOutcome::Failed(e.to_string()),
9410            };
9411            (i, seat, out, interrupted)
9412        });
9413    }
9414    drop(spawned_tx);
9415    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9416    loop {
9417        let joined = tokio::select! {
9418            Some(key) = spawned_rx.recv() => {
9419                // The clock of this seat starts now, not when it queued.
9420                state.seat_spawned(&key);
9421                if let Err(e) = state.save() {
9422                    tracing::warn!("could not persist a seat's start: {e:#}");
9423                }
9424                continue;
9425            }
9426            joined = set.join_next() => joined,
9427        };
9428        let Some(joined) = joined else {
9429            break;
9430        };
9431        let (i, seat, out, interrupted) = match joined {
9432            Ok(v) => v,
9433            // No seat to clear: a panicked task never reported which one it
9434            // was. The defensive sweep below this loop is what stops that
9435            // seat's `active` entry from surviving forever.
9436            Err(e) => {
9437                tracing::error!("agent task panicked: {e}");
9438                continue;
9439            }
9440        };
9441        state.seat_finished(&seat.key);
9442        if interrupted
9443            && node == "implement"
9444            && let Some(label) = state
9445                .candidates
9446                .iter()
9447                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9448                .map(|candidate| candidate.label)
9449        {
9450            state.candidate_preservation.insert(
9451                label,
9452                "interrupted implementation awaiting persistence".into(),
9453            );
9454            state.interrupted_implementations.insert(label);
9455        }
9456        if let AgentOutcome::ResourceBlocked(why) = &out
9457            && state.resource_block.is_none()
9458        {
9459            state.resource_block = Some(crate::run::ResourceBlock {
9460                node: node.to_owned(),
9461                kind: if why.contains("disk pressure") {
9462                    "disk pressure"
9463                } else if why.contains("repository coordination") {
9464                    "repository coordination"
9465                } else {
9466                    "parallel slot"
9467                }
9468                .to_owned(),
9469                detail: why.clone(),
9470                at: Timestamp::now(),
9471            });
9472            let activity = if interrupted {
9473                "agent execution interrupted"
9474            } else {
9475                "no agent was started"
9476            };
9477            state.event("resource", format!("{node}: {activity} - {why}"));
9478        }
9479        record_jobs(state, node, round, &seat.key, &out);
9480        if let Err(e) = state.save() {
9481            tracing::warn!("could not persist a seat's completion: {e:#}");
9482        }
9483        if collected.len() <= i {
9484            collected.resize_with(i + 1, || None);
9485        }
9486        collected[i] = Some((i, seat, out));
9487    }
9488    // Belt-and-braces for the panic branch above: every seat this exact batch
9489    // started shares this `(node, attempt)` pair, and every seat that finished
9490    // normally already cleared itself, so anything left tagged with it here
9491    // can only be a panicked task's leftover. Cleared unconditionally rather
9492    // than left to read as still answering forever.
9493    if state
9494        .active
9495        .values()
9496        .any(|a| a.node == node && a.attempt == attempt)
9497    {
9498        state
9499            .active
9500            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9501        if let Err(e) = state.save() {
9502            tracing::warn!("could not persist the end of a wave: {e:#}");
9503        }
9504    }
9505    // Whether or not the lease above was actually held, several worktrees
9506    // may just have built into the cache with nothing here able to name one
9507    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9508    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9509    // clean it might not have strictly needed; trusting a stale match would
9510    // cost it a wrong answer.
9511    if let Some(cache_dir) = cache
9512        && jobs_had_a_writer
9513    {
9514        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9515    }
9516    if let Some(guard) = cache_guard {
9517        guard.release();
9518    }
9519    collected.into_iter().flatten().collect()
9520}
9521
9522/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9523/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9524/// this is data collection, not the fix-specific completion contract in
9525/// [`Runner::continue_fix_report`], and applies regardless of which node
9526/// asked.
9527///
9528/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9529/// evidence from; `Failed` does not, and correctly contributes nothing — a
9530/// timeout or crash is not itself evidence about a command the seat may have
9531/// started.
9532fn record_jobs(
9533    state: &mut RunState,
9534    node: &str,
9535    round: Option<usize>,
9536    seat: &str,
9537    out: &AgentOutcome,
9538) {
9539    let commands: &[agent::CommandEvidence] = match out {
9540        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9541        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9542    };
9543    let checked_at = Timestamp::now();
9544    for c in commands {
9545        state.jobs.push(JobRecord {
9546            node: node.to_owned(),
9547            round,
9548            seat: seat.to_owned(),
9549            id: c.id.clone(),
9550            description: c.description.clone(),
9551            checked_at,
9552            status: match c.exit_code {
9553                Some(0) => JobStatus::Completed,
9554                Some(_) => JobStatus::Failed,
9555                None => JobStatus::Unknown,
9556            },
9557            exit_code: c.exit_code,
9558            result_summary: c.result_summary.clone(),
9559            source: c.source.clone(),
9560        });
9561    }
9562}
9563
9564/// Is a review round clean, given how many reviewer seats answered against
9565/// how many the round expected?
9566///
9567/// A seat that never answered (timeout, crash, unparsable output) is not a
9568/// seat that read the patch and found nothing — treating it as such is
9569/// exactly the bug this function exists to close. Under the default `block`
9570/// policy a missing seat can never be clean; `warn` still requires the seats
9571/// that *did* answer to have found nothing blocking and verification to be
9572/// green.
9573///
9574/// `quota_missing` narrows that `block` default for exactly one cause of
9575/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9576/// a session limit lifts by the very next round buys nothing — the seat is
9577/// asked again with the same quota — so once every missing seat is accounted
9578/// for by a quota loss (and at least one seat *did* answer, so a decision has
9579/// something to rest on) the round is decided on the panel that could answer,
9580/// same as `warn` would. A panel that lost every seat to quota is not
9581/// decided here: `answered == 0` falls through to the existing `block`
9582/// fallback so a fully collapsed panel still waits rather than landing on no
9583/// review at all.
9584fn round_is_clean(
9585    blocking: usize,
9586    e2e_ok: bool,
9587    answered: usize,
9588    expected: usize,
9589    quota_missing: usize,
9590    policy: IncompleteReviewPolicy,
9591) -> bool {
9592    if blocking != 0 || !e2e_ok {
9593        return false;
9594    }
9595    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9596        return true;
9597    }
9598    answered > 0 && expected - answered <= quota_missing
9599}
9600
9601/// The review loop's own conclusion, derived entirely from its persisted
9602/// round records and the round budget that produced them — never from
9603/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9604/// recomputes the identical answer regardless of what an earlier node in the
9605/// same walk, or a previous walk, did to `status`.
9606///
9607/// `None` while more rounds remain to try, including when review never ran
9608/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9609/// gone clean, or the budget is spent, or the tree has stopped moving (see
9610/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9611///
9612/// - An incomplete panel that raised nothing is missing input, not a
9613///   verified tree — never a hand-off candidate, whatever verification said
9614///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9615/// - Otherwise, green e2e on the last round hands off (see
9616///   [`Runner::stop_reviewing`]); red e2e blocks.
9617///
9618/// A last round whose own verification is still `ResourceBlocked` — magi
9619/// itself never got a command to run, not evidence the patch is broken —
9620/// is neither: this returns `None` for it too, the same as "more rounds
9621/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9622/// handling of that shape) instead of this cheap recomputation guessing a
9623/// verdict a real attempt never produced.
9624pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9625    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9626        return Some(RunStatus::Gating);
9627    }
9628    let last = reviews.last()?;
9629    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9630    if reviews.len() < max_rounds && !stagnant {
9631        return None;
9632    }
9633    if last.incomplete() && last.blocking == 0 {
9634        return Some(RunStatus::Blocked);
9635    }
9636    if last.e2e_status() == E2eStatus::ResourceBlocked {
9637        return None;
9638    }
9639    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9640        RunStatus::Gating
9641    } else {
9642        RunStatus::Blocked
9643    })
9644}
9645
9646/// How long a re-ask may take, given the budget the first attempt had.
9647///
9648/// A `nudged` retry is a request to restate an answer the seat has already
9649/// worked out: it carries no new work, so it does not deserve the original
9650/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9651/// 133 seconds while a third sat for over ten minutes on a resumed session
9652/// holding 410 KB of prior output - and because the retry had inherited the
9653/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9654/// judging round whose other seats were long finished.
9655///
9656/// A quarter of the budget, with a floor so that a deliberately short timeout
9657/// does not collapse to nothing. A retry that re-sends the whole prompt
9658/// (because the seat kept no context) is the original job again, and keeps the
9659/// original budget.
9660pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9661    if nudged {
9662        (full / 4).max(Duration::from_secs(120)).min(full)
9663    } else {
9664        full
9665    }
9666}
9667
9668/// Run a wave and parse each reply, re-asking the seats whose reply was
9669/// unusable.
9670///
9671/// The re-ask is a nudge rather than the whole prompt again when the seat still
9672/// holds its conversation, which is the difference between a cheap retry and
9673/// paying for the entire candidate set twice.
9674///
9675/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9676/// successor in `roster` is handed to it instead of being re-asked: the
9677/// handover is the retry. A seat with no successor left (a single-agent
9678/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9679/// the asks to one seat in one node number at most
9680/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9681/// is asked once (a dropped stream is nudged first), and only the last agent
9682/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9683/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9684/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9685/// other failure stops the chain when the previous agent failed the same way
9686/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9687/// [`has_context`] is false and the job's own full prompt and full budget are
9688/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9689/// intermediate ones are not losses) and is returned as a failure like any
9690/// other absent seat — the caller decides whether the panel still has a
9691/// quorum. An empty `roster` disables handover: failures are nudged as they
9692/// always were, and a quota is simply lost. A reply that fails to parse or
9693/// validate is the prompt's doing and is only ever nudged, never handed over.
9694///
9695/// The returned [`SeatState`] names the agent that answered (or tried last).
9696#[allow(clippy::too_many_arguments)]
9697async fn ask_json_wave<T>(
9698    jobs: Vec<SeatJob>,
9699    sem: Arc<Semaphore>,
9700    retries: usize,
9701    roster: &[AgentSpec],
9702    ctx: &WaveCtx<'_>,
9703    losses: &mut Vec<QuotaLoss>,
9704    state: &mut RunState,
9705    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9706) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9707where
9708    T: serde::de::DeserializeOwned + Send + 'static,
9709{
9710    ask_wave_with(
9711        jobs,
9712        sem,
9713        retries,
9714        roster,
9715        ctx,
9716        losses,
9717        state,
9718        &|text: &str| {
9719            let v = verdict::extract_json::<T>(text)?;
9720            validate(&v)?;
9721            Ok(v)
9722        },
9723    )
9724    .await
9725}
9726
9727/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9728/// a node whose answer is prose (deliberation) shares the same handover,
9729/// failure classification, quota bookkeeping and bounds instead of a copy.
9730///
9731/// A seat handed to another roster agent is sent the job's `handover` prompt
9732/// (when it has one) rather than `prompt`: the new agent has no session, so a
9733/// resume-style prompt would be incomplete. That holds for the handover ask
9734/// and for every nudge to that agent whose `has_context` is false.
9735#[allow(clippy::too_many_arguments)]
9736async fn ask_wave_with<T>(
9737    jobs: Vec<SeatJob>,
9738    sem: Arc<Semaphore>,
9739    retries: usize,
9740    roster: &[AgentSpec],
9741    ctx: &WaveCtx<'_>,
9742    losses: &mut Vec<QuotaLoss>,
9743    state: &mut RunState,
9744    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9745) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9746where
9747    T: Send + 'static,
9748{
9749    let n = jobs.len();
9750    let originals: Vec<SeatJob> = jobs;
9751    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9752    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9753    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9754    // answer, N once it has gone through N nudges. Read back once this
9755    // returns, so a caller building a history record (`ReviewRecord`) can
9756    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9757    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9758    // field's own doc.
9759    let mut nudges: Vec<usize> = vec![0; n];
9760    // The agent now occupying each seat, the ids it has already been through,
9761    // where in the roster the walk began, the class of the last failure, and
9762    // the stem word of a handover not yet asked (full prompt, full budget).
9763    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9764    let mut tried: Vec<BTreeSet<String>> = specs
9765        .iter()
9766        .map(|s| BTreeSet::from([s.id.clone()]))
9767        .collect();
9768    let starts: Vec<usize> = specs
9769        .iter()
9770        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9771        .collect();
9772    let carry = ctx.carry_seats && !roster.is_empty();
9773    // Carried across rounds: ids that failed the seat earlier, and how the
9774    // last failure went (so a repeat of it is not handed over again).
9775    let carried: Vec<BTreeSet<String>> = originals
9776        .iter()
9777        .map(|j| {
9778            state
9779                .seat_history
9780                .get(&j.seat.key)
9781                .filter(|_| carry)
9782                .map(|h| h.failed.clone())
9783                .unwrap_or_default()
9784        })
9785        .collect();
9786    let mut prev: Vec<Option<FailClass>> = originals
9787        .iter()
9788        .map(|j| {
9789            state
9790                .seat_history
9791                .get(&j.seat.key)
9792                .filter(|_| carry)
9793                .and_then(|h| h.last_fail.clone())
9794        })
9795        .collect();
9796    let next_agent =
9797        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9798            let others: BTreeSet<String> = specs
9799                .iter()
9800                .enumerate()
9801                .filter(|(j, _)| *j != i)
9802                .map(|(_, s)| s.id.clone())
9803                .collect();
9804            pick_successor(
9805                roster,
9806                starts[i],
9807                tried,
9808                carry.then(|| &carried[i]),
9809                &others,
9810            )
9811            .cloned()
9812        };
9813    let mut fresh: Vec<Option<String>> = vec![None; n];
9814    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9815    let mut pending: Vec<usize> = (0..n).collect();
9816
9817    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
9818    // asks: an agent with a successor is asked once and handed over, and only
9819    // a seat with no successor spends `retries` nudges on the same agent. This
9820    // only guarantees the loop's own termination whatever those say.
9821    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9822    for round in 0..max_rounds {
9823        if pending.is_empty() {
9824            break;
9825        }
9826        let mut batch = Vec::with_capacity(pending.len());
9827        let mut renudged: Vec<&str> = Vec::new();
9828        for &i in &pending {
9829            let src = &originals[i];
9830            // A seat now held by another agent than the job named has no
9831            // session of its own: it gets the full-context prompt whenever
9832            // it is asked in full (the handover ask, a nudge it cannot
9833            // resume).
9834            let full: &str = match &src.handover {
9835                Some(h) if specs[i].id != src.spec.id => h,
9836                _ => &src.prompt,
9837            };
9838            // The prompt and the budget are one decision: a nudge restates
9839            // finished work, a re-sent prompt redoes it.
9840            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9841                (
9842                    full.to_owned(),
9843                    src.timeout,
9844                    format!("{}-{word}-{}", src.stem, specs[i].id),
9845                )
9846            } else if nudges[i] == 0 {
9847                (full.to_owned(), src.timeout, src.stem.clone())
9848            } else {
9849                renudged.push(src.seat.key.as_str());
9850                let why = done[i]
9851                    .as_ref()
9852                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
9853                    .unwrap_or_else(|| "no parsable answer".to_owned());
9854                let nudge = prompt::nudge(&why);
9855                let nudged = has_context(&specs[i], &seats[i], src.sessions);
9856                let prompt = if nudged {
9857                    nudge
9858                } else {
9859                    format!("{full}\n\n---\n\n{nudge}")
9860                };
9861                (
9862                    prompt,
9863                    retry_budget(src.timeout, nudged),
9864                    format!("{}-retry{}", src.stem, nudges[i]),
9865                )
9866            };
9867            batch.push(SeatJob {
9868                spec: specs[i].clone(),
9869                seat: seats[i].clone(),
9870                cwd: src.cwd.clone(),
9871                prompt,
9872                timeout,
9873                allow_write: src.allow_write,
9874                sessions: src.sessions,
9875                artifacts: src.artifacts.clone(),
9876                stem,
9877                handover: None,
9878            });
9879        }
9880
9881        if !renudged.is_empty() {
9882            state.event(
9883                ctx.node,
9884                format!("retry {round}: re-asking {}", renudged.join(", ")),
9885            );
9886        }
9887        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9888        let mut still = Vec::new();
9889        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9890            seats[i] = seat;
9891            // No agent was asked: nothing to hand over, nudge, remember or
9892            // charge. The caller sees `resource_block` and stops the node.
9893            if let AgentOutcome::ResourceBlocked(why) = &out {
9894                done[i] = Some(Err(anyhow::anyhow!("{why}")));
9895                continue;
9896            }
9897            let class = FailClass::of(&out);
9898            // A dropped stream is nudged first (the conversation is still
9899            // there to pick up); only a seat whose nudges are spent hands over.
9900            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9901                && nudges[i] < retries
9902                && !roster.is_empty();
9903            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9904                let next = should_hand_over(prev[i].as_ref(), &cur)
9905                    .then(|| next_agent(i, &tried[i], &specs))
9906                    .flatten();
9907                if carry {
9908                    let h = state
9909                        .seat_history
9910                        .entry(originals[i].seat.key.clone())
9911                        .or_default();
9912                    h.failed.insert(specs[i].id.clone());
9913                    h.last_fail = Some(cur.clone());
9914                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9915                        h.last_ok = None;
9916                    }
9917                    // Saved before the next agent is asked, so a restart in
9918                    // between does not forget who failed.
9919                    if let Err(e) = state.save() {
9920                        tracing::warn!("could not persist a seat's failure history: {e:#}");
9921                    }
9922                }
9923                if let Some(next) = next {
9924                    record_handover(
9925                        state,
9926                        ctx.node,
9927                        &originals[i].seat.key,
9928                        &specs[i].id,
9929                        &next.id,
9930                        &cur,
9931                        &fail_reason(&out),
9932                    );
9933                    tried[i].insert(next.id.clone());
9934                    prev[i] = Some(cur.clone());
9935                    seats[i] =
9936                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9937                    specs[i] = next;
9938                    fresh[i] = Some(cur.stem_word().to_owned());
9939                    nudges[i] = 0;
9940                    done[i] = Some(Err(anyhow::anyhow!(
9941                        "handed over after: {}",
9942                        fail_reason(&out)
9943                    )));
9944                    still.push(i);
9945                    continue;
9946                }
9947            }
9948            if carry
9949                && !nudge_first
9950                && let Some(cur) = class.clone()
9951            {
9952                // The chain ended here (no successor, or a repeated failure).
9953                let h = state
9954                    .seat_history
9955                    .entry(originals[i].seat.key.clone())
9956                    .or_default();
9957                h.failed.insert(specs[i].id.clone());
9958                h.last_fail = Some(cur);
9959            }
9960            let parsed = match out {
9961                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9962                AgentOutcome::Quota(o) => {
9963                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9964                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9965                }
9966                // Not a parseable answer: the nudge loop re-asks it, which is
9967                // exactly what a dropped stream needs. Just don't hand its raw
9968                // error JSON to `extract_json`.
9969                AgentOutcome::Dropped(o) => {
9970                    let why = o
9971                        .dropped
9972                        .as_ref()
9973                        .map(|d| d.why.as_str())
9974                        .unwrap_or("the CLI ended the stream without delivering its answer");
9975                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9976                }
9977                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9978                    Err(anyhow::anyhow!(e))
9979                }
9980            };
9981            let quota = class == Some(FailClass::Quota);
9982            let failed = parsed.is_err();
9983            done[i] = Some(parsed);
9984            if carry && !failed {
9985                let h = state
9986                    .seat_history
9987                    .entry(originals[i].seat.key.clone())
9988                    .or_default();
9989                h.failed.remove(&specs[i].id);
9990                h.last_ok = Some(specs[i].id.clone());
9991                h.last_fail = None;
9992            }
9993            // Do not re-ask a rate-limited seat (quota) — a retry is known to
9994            // fail the same way; and never re-ask a seat that already parsed.
9995            // A failed agent that still has a successor is not re-asked
9996            // either: the handover was its remedy and has just been refused
9997            // (the chain stops on a repeated failure class). A seat with no
9998            // successor left (a single-agent roster, the roster's tail, or an
9999            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10000            let agent_failure = class.is_some()
10001                && !nudge_first
10002                && !roster.is_empty()
10003                && next_agent(i, &tried[i], &specs).is_some();
10004            if failed && !quota && !agent_failure && nudges[i] < retries {
10005                nudges[i] += 1;
10006                still.push(i);
10007            }
10008        }
10009        pending = still;
10010    }
10011
10012    // One loss per seat whose chain ended on a quota: the intermediate ones
10013    // were absorbed by a handover and are not losses.
10014    for (i, q) in last_quota.into_iter().enumerate() {
10015        if let Some(reset) = q {
10016            losses.push(QuotaLoss {
10017                seat: originals[i].seat.key.clone(),
10018                node: ctx.node.to_owned(),
10019                at: Timestamp::now(),
10020                reset,
10021            });
10022        }
10023    }
10024
10025    seats
10026        .into_iter()
10027        .zip(done)
10028        .zip(nudges)
10029        .map(|((seat, res), attempts)| {
10030            (
10031                seat,
10032                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10033                attempts,
10034            )
10035        })
10036        .collect()
10037}
10038
10039/// Did the wave end without asking any agent (see [`block_wave`])?
10040fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10041    results
10042        .iter()
10043        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10044}
10045
10046/// Settle a wave that never spawned anything: every job answers
10047/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10048/// else about the seat (session, history, quota) is touched.
10049fn block_wave(
10050    jobs: Vec<SeatJob>,
10051    state: &mut RunState,
10052    node: &str,
10053    attempt: usize,
10054    why: &str,
10055) -> Vec<(usize, SeatState, AgentOutcome)> {
10056    state.resource_wait = None;
10057    state
10058        .active
10059        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10060    if let Err(e) = state.save() {
10061        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10062    }
10063    jobs.into_iter()
10064        .enumerate()
10065        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10066        .collect()
10067}
10068
10069/// Acquire the shared build cache's lease, waiting out contention within
10070/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10071/// unbounded wait is never acceptable).
10072///
10073/// A first, non-blocking check happens before ever waiting; if it finds the
10074/// lease busy, that fact is logged as a `verify` event *and* flushed with
10075/// [`RunState::save`] immediately — not only once the wait finally succeeds
10076/// or gives up — so a `magi show` run by a different process while this one
10077/// is still waiting reads a `run.json` that says so, rather than whatever it
10078/// looked like before the wait started. The same applies to the terminal
10079/// failure: logged and saved before this returns `Err`, so a caller that
10080/// could not get the lease at all still leaves a legible record of why.
10081async fn acquire_cache_lease(
10082    state: &mut RunState,
10083    cache_dir: &Path,
10084    owner: &crate::cache::Owner,
10085    budget: Duration,
10086    context: &str,
10087    cancel: &(dyn Fn() -> bool + Sync),
10088) -> Result<Option<crate::cache::Guard>> {
10089    let home = crate::run::home();
10090    let started = Instant::now();
10091    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10092        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10093        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10094        Err(e) => {
10095            state.event(
10096                "verify",
10097                format!("{context}: could not check the shared build cache: {e:#}"),
10098            );
10099            if let Err(e2) = state.save() {
10100                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10101            }
10102            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10103        }
10104    };
10105    state.event(
10106        "verify",
10107        format!(
10108            "{context}: waiting for the shared build cache at {} ({})",
10109            cache_dir.display(),
10110            busy.describe()
10111        ),
10112    );
10113    // Seats of this node are waiting, not running: say so, so the countdown a
10114    // reader sees only ever measures an agent that has been spawned.
10115    state.resource_wait = Some(crate::run::ResourceWait {
10116        node: context.to_owned(),
10117        kind: "shared build cache".to_owned(),
10118        holder: busy.describe(),
10119        since: Timestamp::now(),
10120        limit_secs: budget.as_secs(),
10121    });
10122    if let Err(e) = state.save() {
10123        tracing::warn!("could not persist a cache wait: {e:#}");
10124    }
10125    let remaining = budget.saturating_sub(started.elapsed());
10126    let waited = crate::cache::wait_for_unless(
10127        &home,
10128        cache_dir,
10129        owner,
10130        remaining,
10131        Duration::from_secs(5),
10132        cancel,
10133    )
10134    .await;
10135    state.resource_wait = None;
10136    match waited {
10137        Ok(g) => Ok(g),
10138        Err(e) => {
10139            state.event("verify", format!("{context}: {e:#}"));
10140            if let Err(e2) = state.save() {
10141                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10142            }
10143            Err(e)
10144        }
10145    }
10146}
10147
10148/// The lease state could not be read at all (unreadable lease file, a
10149/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10150/// known to be contended, so a wave degrades to running without a shared cache
10151/// instead of stopping on a hold that [`crate::triage`] could never release.
10152#[derive(Debug)]
10153struct CacheCheckError(String);
10154
10155impl std::fmt::Display for CacheCheckError {
10156    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10157        f.write_str(&self.0)
10158    }
10159}
10160
10161impl std::error::Error for CacheCheckError {}
10162
10163/// Run `body` — a verify command batch — while holding the shared build
10164/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10165/// never interleave with another borrower's build against the same
10166/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10167/// timeout, or a human's own `magi review`. See the `cache` module doc for
10168/// why this matters more than Cargo's own per-target locking covers — two
10169/// *different* worktrees building the same package name/version into one
10170/// cache directory is a staleness bug, not a lock contention one.
10171///
10172/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10173/// agent waves do. Repository scheduling also has its own bounded,
10174/// cancellable resource-wait policy. Contention must not consume the
10175/// verification budget: `body` receives the full `budget` after both leases
10176/// are acquired and the cache is fresh. Without a shared cache, only
10177/// repository coordination is acquired before handing over the full budget.
10178///
10179/// A lease that cannot be acquired within the resource wait is reported as a single
10180/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10181/// verification — the same shape a spawn failure already takes in
10182/// [`run_commands`], so a caller need not special-case it.
10183#[allow(clippy::too_many_arguments)]
10184async fn with_cache_lease<'s, F, Fut>(
10185    state: &'s mut RunState,
10186    cache_dir: Option<&Path>,
10187    node: &str,
10188    seat: &str,
10189    worktree: &Path,
10190    head: &str,
10191    budget: Duration,
10192    context: &str,
10193    wait: &crate::repo_lock::WaitPolicy,
10194    body: F,
10195) -> Result<(Vec<CommandOutcome>, bool)>
10196where
10197    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10198    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10199{
10200    // Match editing waves: cache first, repository second. In particular,
10201    // pre_gate must never hold the repository while waiting for a wave's cache.
10202    let started = Instant::now();
10203    let home = crate::run::home();
10204    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10205    let guard = if let Some(cache_dir) = cache_dir {
10206        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10207        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10208            wait.cancelled()
10209        })
10210        .await
10211        .and_then(|g| {
10212            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10213        }) {
10214            Ok(g) => Some(g),
10215            Err(e) => {
10216                return Ok((
10217                    vec![CommandOutcome {
10218                        command: "(waiting for the shared build cache)".into(),
10219                        code: None,
10220                        output_tail: e.to_string(),
10221                        duration_ms: started.elapsed().as_millis() as u64,
10222                        resource_blocked: true,
10223                    }],
10224                    false,
10225                ));
10226            }
10227        }
10228    } else {
10229        None
10230    };
10231    let _coordination = coordinate_operation(state, worktree, wait).await?;
10232    if let Some(cache_dir) = cache_dir {
10233        let identity = crate::cache::Identity::new(worktree, head);
10234        if let Err(e) = crate::cache::ensure_fresh(
10235            &home,
10236            cache_dir,
10237            &identity,
10238            &state.config.verify.cargo_manifests(),
10239        ) {
10240            state.event(
10241                "verify",
10242                format!(
10243                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10244                    worktree.display(),
10245                    short(head)
10246                ),
10247            );
10248            if let Some(guard) = guard {
10249                guard.release();
10250            }
10251            return Ok((
10252                vec![CommandOutcome {
10253                    command: "(confirming the shared build cache is fresh)".into(),
10254                    code: None,
10255                    output_tail: e.to_string(),
10256                    duration_ms: started.elapsed().as_millis() as u64,
10257                    resource_blocked: true,
10258                }],
10259                false,
10260            ));
10261        }
10262    }
10263    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10264    if !timed_out_pids.is_empty() {
10265        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10266    }
10267    // Drop repository coordination before releasing the cache, reversing the
10268    // acquisition order. A cancelled/error future also drops both guards.
10269    drop(_coordination);
10270    if let Some(guard) = guard {
10271        guard.release();
10272    }
10273    Ok((outcomes, retried))
10274}
10275
10276/// Poll `pids` — commands [`run_commands`] reports as still running when its
10277/// own timeout elapsed — until every one is confirmed gone, or
10278/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10279///
10280/// Real confirmation where confirmation is possible, not a substitute for
10281/// full process-tree observation: a grandchild the timed-out process spawned
10282/// and that survives independently of it is invisible to a pid check the
10283/// same way it always was, and continuing to observe and collect *that*
10284/// stays a different piece of work with its own owner. This only narrows a
10285/// fixed blind wait into an actual check of the pids this process does know
10286/// about.
10287async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10288    wait_for_pids_with(
10289        pids,
10290        crate::proc::pid_alive,
10291        LEASE_RELEASE_POLL,
10292        LEASE_RELEASE_MAX_WAIT,
10293    )
10294    .await;
10295}
10296
10297/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10298/// interval and ceiling supplied by the caller, so the polling *logic* -
10299/// returns as soon as every pid reports dead, gives up at the ceiling
10300/// otherwise - is testable on millisecond durations without asking the real
10301/// OS about a pid at all.
10302async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10303    pids: &[u32],
10304    alive: F,
10305    poll: Duration,
10306    max_wait: Duration,
10307) {
10308    let deadline = Instant::now() + max_wait;
10309    loop {
10310        if pids.iter().all(|&pid| !alive(pid)) {
10311            return;
10312        }
10313        if Instant::now() >= deadline {
10314            return;
10315        }
10316        tokio::time::sleep(poll).await;
10317    }
10318}
10319
10320/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10321/// admission that it could not even get a verify command to run, as opposed
10322/// to evidence the command actually produced? A caller that would otherwise
10323/// read a resource-blocked outcome as a red command must check this first:
10324/// see [`Runner::gate`], which retries rather than records `Blocked` when
10325/// this is true.
10326fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10327    outcomes.iter().any(|o| o.resource_blocked)
10328}
10329
10330/// What [`Runner::gate_fix_round`] decided.
10331pub(crate) enum GateFix {
10332    /// The tree changed and `verify.e2e` is still green: run the gate again.
10333    Retry,
10334    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10335    /// last failure stands and the run ends blocked.
10336    Stop,
10337    /// `verify.e2e` could not run after the fix (magi's own contention):
10338    /// decide nothing now, a later reentry retries.
10339    Defer,
10340}
10341
10342/// Is every red command in `outcomes` an ordinary failure the code could
10343/// explain: it ran, exited non-zero, and said something?
10344///
10345/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10346/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10347/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10348/// what the command is stays the gate's business.
10349fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10350    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10351    red.peek().is_some()
10352        && red.all(|o| {
10353            !o.resource_blocked
10354                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10355                && !o.output_tail.trim().is_empty()
10356        })
10357}
10358
10359/// Describe one verify command's outcome for the event log, distinguishing a
10360/// build/link failure — the toolchain never produced a binary to run — from
10361/// an actual test failure, since only the latter is a verdict on the patch.
10362fn e2e_outcome_label(o: &CommandOutcome) -> String {
10363    if o.ok() {
10364        return "pass".to_owned();
10365    }
10366    let reason = if o.build_failed() {
10367        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10368    } else {
10369        format!("FAIL ({:?})", o.code)
10370    };
10371    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10372}
10373
10374/// Run `verify.e2e`, retrying once if the first attempt could not build or
10375/// link — a build/link failure is frequently a race against a shared
10376/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10377/// `verify` event per command, tagged with `context` (normally `"round N"`)
10378/// so the two call sites that need this — the ordinary per-round leg in
10379/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10380/// it will ever call a round green — read identically in the event log.
10381async fn run_e2e_with_retry(
10382    state: &mut RunState,
10383    shell: &[String],
10384    commands: &[String],
10385    worktree: &Path,
10386    timeout: Duration,
10387    context: &str,
10388) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10389    let (mut e2e, mut timed_out_pids) = run_commands(
10390        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10391    )
10392    .await;
10393    for o in &e2e {
10394        state.event(
10395            "verify",
10396            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10397        );
10398    }
10399    // A build/link failure is not a verdict on the patch — it is frequently a
10400    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10401    // one retry before letting a red like that decide the round.
10402    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10403    if verify_retried {
10404        state.event(
10405            "verify",
10406            format!(
10407                "{context}: verify could not build/link, not a test result — retrying once \
10408                 before concluding"
10409            ),
10410        );
10411        let retried = run_commands(
10412            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10413        )
10414        .await;
10415        e2e = retried.0;
10416        // Both attempts' timeouts matter, not just the last one: the first
10417        // attempt's descendants may still be alive alongside the retry's.
10418        timed_out_pids.extend(retried.1);
10419        for o in &e2e {
10420            state.event(
10421                "verify",
10422                format!(
10423                    "{context}: retry `{}` -> {}",
10424                    o.command,
10425                    e2e_outcome_label(o)
10426                ),
10427            );
10428        }
10429    }
10430    (e2e, verify_retried, timed_out_pids)
10431}
10432
10433/// Run configured shell commands in `cwd`, in order. The second element is
10434/// the pid of every command that hit `timeout` and was still running when
10435/// this stopped waiting on it (best-effort: `None` when the platform did not
10436/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10437/// that releases a shared resource afterward needs to know.
10438///
10439/// Records `task` into [`RunState::active`] at every command boundary
10440/// (`RunState::task_command`) and clears it once the whole list has run
10441/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10442/// for minutes with no seat and no output of its own to show for it (see
10443/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10444/// apart from "not yet run" without this), and this is the only place that
10445/// knows which command is running right now and how many are left. Three
10446/// saves per command — start, not per second — matching the same "only at a
10447/// boundary" rule [`wave`] already follows for seats.
10448#[allow(clippy::too_many_arguments)]
10449async fn run_commands(
10450    state: &mut RunState,
10451    node: &str,
10452    task: &str,
10453    attempt: usize,
10454    shell: &[String],
10455    commands: &[String],
10456    cwd: &Path,
10457    timeout: Duration,
10458) -> (Vec<CommandOutcome>, Vec<u32>) {
10459    if commands.is_empty() {
10460        // Nothing to mark as running and nothing to clear — an empty list
10461        // means "not configured", and touching `active` (or the disk) over
10462        // that would be a write for every round of a repo with no
10463        // `verify.e2e` / `verify.gate` commands at all.
10464        return (Vec::new(), Vec::new());
10465    }
10466    let mut out = Vec::new();
10467    let mut timed_out_pids = Vec::new();
10468    let total = commands.len();
10469    for (idx, command) in commands.iter().enumerate() {
10470        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10471        if let Err(e) = state.save() {
10472            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10473        }
10474        let started = Instant::now();
10475        let mut cmd = tokio::process::Command::new(&shell[0]);
10476        cmd.quiet();
10477        cmd.args(&shell[1..])
10478            .arg(command)
10479            .current_dir(cwd)
10480            .stdin(std::process::Stdio::null())
10481            .stdout(std::process::Stdio::piped())
10482            .stderr(std::process::Stdio::piped())
10483            .kill_on_drop(true);
10484        let mut outputs = Vec::new();
10485        let protected = (|| -> Result<()> {
10486            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10487                outputs.push(guard);
10488            }
10489            if let Some(cache) = state.config.cache_dir() {
10490                outputs.push(crate::retention::OutputLease::cache(
10491                    &crate::run::home(),
10492                    &cache,
10493                    false,
10494                )?);
10495            }
10496            Ok(())
10497        })();
10498        if let Err(e) = protected {
10499            out.push(CommandOutcome {
10500                command: command.clone(),
10501                code: None,
10502                output_tail: format!("output ownership unavailable: {e:#}"),
10503                duration_ms: 0,
10504                resource_blocked: true,
10505            });
10506            continue;
10507        }
10508        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10509        let (code, body) = match spawned {
10510            Ok(mut running) => {
10511                let pid = running.child().id();
10512                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10513                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10514                let mut paths = vec![cwd.to_owned()];
10515                if let Some(cache) = state.config.cache_dir() {
10516                    paths.push(cache);
10517                }
10518                let mut pressure = None;
10519                let status = tokio::select! {
10520                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10521                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10522                        pressure = Some(reason); None
10523                    }
10524                };
10525                let (code, failure) = match status {
10526                    Some(Ok(Ok(status))) => (
10527                        status.code(),
10528                        status
10529                            .code()
10530                            .is_none()
10531                            .then(|| format!("command terminated: {status}")),
10532                    ),
10533                    Some(Ok(Err(error))) => {
10534                        (None, Some(format!("failed waiting for command: {error}")))
10535                    }
10536                    Some(Err(_)) => {
10537                        // Leased cleanup confirms process-tree death itself.
10538                        // Unleased commands still need the caller's PID check
10539                        // after kill-on-drop, before releasing cache ownership.
10540                        if !running.coordinated()
10541                            && let Some(pid) = pid
10542                        {
10543                            timed_out_pids.push(pid);
10544                        }
10545                        (
10546                            None,
10547                            Some(format!("timed out after {}s", timeout.as_secs())),
10548                        )
10549                    }
10550                    None => (None, None),
10551                };
10552                running.finish().await;
10553                if let Some(reason) = pressure {
10554                    state.resource_block = Some(crate::run::ResourceBlock {
10555                        node: node.to_owned(),
10556                        kind: "disk pressure".to_owned(),
10557                        detail: reason.clone(),
10558                        at: Timestamp::now(),
10559                    });
10560                    out.push(CommandOutcome {
10561                        command: command.clone(),
10562                        code: None,
10563                        output_tail: reason,
10564                        duration_ms: started.elapsed().as_millis() as u64,
10565                        resource_blocked: true,
10566                    });
10567                    break;
10568                }
10569                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10570                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10571                if let Some(failure) = failure {
10572                    body.push('\n');
10573                    body.push_str(&failure);
10574                }
10575                (code, body)
10576            }
10577            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10578        };
10579        out.push(CommandOutcome {
10580            command: command.clone(),
10581            code,
10582            output_tail: tail(&body, OUTPUT_TAIL),
10583            duration_ms: started.elapsed().as_millis() as u64,
10584            resource_blocked: false,
10585        });
10586    }
10587    state.task_finished(task);
10588    if let Err(e) = state.save() {
10589        tracing::warn!("could not persist the end of {task}: {e:#}");
10590    }
10591    (out, timed_out_pids)
10592}
10593
10594/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10595/// section (`report::run`) and in the `merge` event this node records — for
10596/// the operator to run by hand.
10597///
10598/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10599/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10600/// merge commits", or "require linear history") rejects the push a `--no-ff`
10601/// merge would produce, which is exactly the guidance this function replaces.
10602/// `message`'s first line becomes the squash commit's subject, matching the
10603/// note `report::run` prints alongside this command — see that function for
10604/// why an explicit subject is not optional there.
10605fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10606    let repo = repo.display();
10607    match style {
10608        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10609        MergeStyle::Squash => {
10610            // The subject sits inside double quotes, and a title an agent
10611            // wrote may carry the characters that break out of them.
10612            let subject = message
10613                .lines()
10614                .next()
10615                .unwrap_or(branch)
10616                .replace(['\\', '"', '$', '`'], "");
10617            format!(
10618                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10619            )
10620        }
10621        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10622    }
10623}
10624
10625/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10626/// under the hood, rejects a `title` over 256 characters and the whole
10627/// command fails — no PR at all, for a run whose body was otherwise fine
10628/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10629/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10630/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10631/// this bound. It is a margin, not a guarantee — a title packed
10632/// with multi-unit characters could still in principle land close to the
10633/// edge, but a real task title's occasional emoji or accented letter fits
10634/// comfortably inside it.
10635const PR_TITLE_MAX: usize = 240;
10636
10637/// A pull request title from the opening line of `text`, or `None` when that
10638/// line is not English (GitHub text is) or has no letters.
10639///
10640/// A line within `max` is kept as is. A longer one is cut at the end of its
10641/// first sentence when that falls inside `max`, else at a word boundary with a
10642/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10643/// make every merely-truncated title look non-English). The language check
10644/// runs on the kept text before any mark is added, so only what GitHub will
10645/// show is judged: an English opening followed by non-ASCII far past the cut
10646/// still passes.
10647fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10648    let line = queue::first_line(text)?;
10649    let chars: Vec<char> = line.chars().collect();
10650    let (kept, mark) = if chars.len() <= max {
10651        (line.to_owned(), "")
10652    } else if let Some(end) = sentence_end(&chars, max) {
10653        (chars[..end].iter().collect::<String>(), "")
10654    } else {
10655        let room = max.saturating_sub(3);
10656        // Cut at the last space inside the room; when the char just past the
10657        // room is a space the room already ends on a word.
10658        let cut = if chars[room].is_whitespace() {
10659            room
10660        } else {
10661            chars[..room]
10662                .iter()
10663                .rposition(|c| c.is_whitespace())
10664                .unwrap_or(room)
10665        };
10666        let head: String = chars[..cut].iter().collect();
10667        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10668        (head.to_owned(), "...")
10669    };
10670    if kept.is_empty() {
10671        return None;
10672    }
10673    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10674        return None;
10675    }
10676    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10677        return None;
10678    }
10679    Some(format!("{kept}{mark}"))
10680}
10681
10682/// The char length of the first sentence of `chars` when it ends within `max`
10683/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10684/// abbreviations so `e.g. foo` does not end a title early.
10685fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10686    const MIN: usize = 20;
10687    for i in MIN..max.min(chars.len()) {
10688        if !matches!(chars[i], '.' | '!' | '?') {
10689            continue;
10690        }
10691        let Some(&next) = chars.get(i + 1) else {
10692            continue;
10693        };
10694        if !next.is_whitespace() {
10695            continue;
10696        }
10697        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10698        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10699            continue;
10700        }
10701        let word: String = chars[..i]
10702            .iter()
10703            .rev()
10704            .take_while(|c| !c.is_whitespace())
10705            .collect::<Vec<_>>()
10706            .into_iter()
10707            .rev()
10708            .collect();
10709        let word = word.to_ascii_lowercase();
10710        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10711            continue;
10712        }
10713        let end = chars[..i]
10714            .iter()
10715            .rposition(|c| !c.is_whitespace())
10716            .map_or(i, |p| p + 1);
10717        return Some(end);
10718    }
10719    None
10720}
10721
10722/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10723/// change: a title and a body describing what was *implemented*, not the task
10724/// that asked for it. A task reads as a request; a reader of the merged
10725/// history wants the change.
10726struct PrMessage {
10727    title: String,
10728    body: String,
10729}
10730
10731impl PrMessage {
10732    /// Title, blank line, body. The first line is the squash/merge commit
10733    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10734    /// has to stay one sensible line.
10735    fn commit_message(&self) -> String {
10736        format!("{}\n\n{}", self.title, self.body)
10737    }
10738}
10739
10740/// The text after a leading `TITLE:` (any case) on `line`.
10741fn title_marker(line: &str) -> Option<&str> {
10742    let line = line.trim();
10743    let head = line.get(..6)?;
10744    head.eq_ignore_ascii_case("title:")
10745        .then(|| line[6..].trim())
10746}
10747
10748/// The implementer's own one-line title: the `TITLE:` line the implement
10749/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10750/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10751/// source, and a title that says as much is refused here too.
10752fn summary_title(summary: &str) -> Option<String> {
10753    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10754    let raw = title_marker(first)?;
10755    if raw.is_empty() {
10756        return None;
10757    }
10758    let title = queue::title_from(raw, PR_TITLE_MAX);
10759    let lower = title.to_ascii_lowercase();
10760    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10761        return None;
10762    }
10763    Some(title)
10764}
10765
10766/// How `open_review`'s instruction begins; see [`landing_title`].
10767const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10768
10769/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10770pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10771
10772/// Does this run review work that already existed, rather than implement a
10773/// task? Runs recorded before `reviewed_commits` existed carry only the
10774/// candidate marker.
10775fn is_review_run(state: &RunState) -> bool {
10776    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10777}
10778
10779/// The title of a review-only run: the subject of the oldest commit under
10780/// review. Later commits are usually fixups, and `instruction` is the review
10781/// prompt, which says nothing about the change. GitHub text is English, so a
10782/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10783fn review_title(state: &RunState) -> Option<String> {
10784    english_subject(
10785        state.reviewed_commits.as_ref()?.first()?,
10786        state.config.graph.github_english(),
10787    )
10788}
10789
10790/// `raw` as a pull request title, or `None` when it is blank, not English
10791/// (GitHub text is), or one of magi's own candidate commit subjects.
10792fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10793    let raw = raw.trim();
10794    if raw.is_empty() {
10795        return None;
10796    }
10797    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10798        return None;
10799    }
10800    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10801        return None;
10802    }
10803    let title = queue::title_from(raw, PR_TITLE_MAX);
10804    let lower = title.to_ascii_lowercase();
10805    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10806        return None;
10807    }
10808    Some(title)
10809}
10810
10811/// What a review-only run's branch says about itself, read at the moment the
10812/// pull request is opened.
10813#[derive(Debug, Clone, PartialEq, Eq)]
10814struct BranchFacts {
10815    /// `(subject, body)` of each commit, oldest first.
10816    commits: Vec<(String, String)>,
10817    /// Trimmed `git diff --stat`.
10818    stat: String,
10819}
10820
10821/// Longest diff stat shown: this many file lines plus the summary line.
10822const STAT_FILE_LINES: usize = 25;
10823/// Cap on the commit list, well inside GitHub's 65536-character body limit.
10824const COMMITS_MAX_CHARS: usize = 20_000;
10825
10826/// Read the commits and diff stat of `base..branch`. `None` when git cannot
10827/// say or finds nothing, so the caller falls back to what the run recorded.
10828async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10829    let commits = git::commit_log(repo, base, branch).await.ok()?;
10830    if commits.is_empty() {
10831        return None;
10832    }
10833    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10834    let lines: Vec<&str> = stat.lines().collect();
10835    let stat = if lines.len() > STAT_FILE_LINES + 1 {
10836        let omitted = lines.len() - 1 - STAT_FILE_LINES;
10837        let more = format!(" ... {omitted} more file(s)");
10838        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10839        kept.push(&more);
10840        kept.push(lines[lines.len() - 1]);
10841        kept.join("\n")
10842    } else {
10843        lines.join("\n")
10844    };
10845    Some(BranchFacts { commits, stat })
10846}
10847
10848/// Defang what would break the surrounding markdown: a closing `</details>`
10849/// and a code fence.
10850fn markdown_safe(text: &str) -> String {
10851    text.replace("</details>", "&lt;/details&gt;")
10852        .replace("\x60\x60\x60", "~~~")
10853}
10854
10855fn neutral_title(state: &RunState, winner: char) -> String {
10856    format!(
10857        "chore: land candidate {} of run {}",
10858        winner.to_ascii_uppercase(),
10859        state.id
10860    )
10861}
10862
10863/// Reuse the graph's bounded fixer without starting another graph or driver.
10864/// Copy the working state back even on errors so spent rounds remain visible.
10865pub(crate) async fn repair_landing_gate(
10866    state: &mut RunState,
10867    winner: &Candidate,
10868    outcomes: &[CommandOutcome],
10869) -> Result<GateFix> {
10870    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10871        return Ok(GateFix::Stop);
10872    }
10873    let mut runner = Runner {
10874        roles: state.config.resolve_roles()?,
10875        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10876        state: state.clone(),
10877        pause: Pause::new(),
10878        interrupt: Pause::new(),
10879    };
10880    let result = runner.gate_fix_round(winner, outcomes).await;
10881    *state = runner.state;
10882    result
10883}
10884
10885/// Recheck a repair's e2e requirement across failures, resource deferrals and
10886/// process interruptions. A successful result only applies to its exact SHA.
10887pub(crate) async fn check_gate_repair_e2e(
10888    state: &mut RunState,
10889    winner: &Candidate,
10890    wait: &crate::repo_lock::WaitPolicy,
10891) -> Result<GateFix> {
10892    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10893        return Ok(GateFix::Retry);
10894    }
10895    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10896    if state
10897        .gate_repair_e2e
10898        .as_ref()
10899        .is_some_and(|r| r.head == head && r.passed)
10900    {
10901        return Ok(GateFix::Retry);
10902    }
10903    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10904        head: head.clone(),
10905        passed: false,
10906        outcomes: Vec::new(),
10907    });
10908    state.save()?;
10909    let commands = state.config.verify.e2e.clone();
10910    let outcomes = if commands.is_empty() {
10911        Vec::new()
10912    } else {
10913        let shell = state.config.shell();
10914        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10915        let cache_dir = state.config.cache_dir();
10916        let (outcomes, _) = with_cache_lease(
10917            state,
10918            cache_dir.as_deref(),
10919            "e2e",
10920            "e2e",
10921            &winner.worktree,
10922            &head,
10923            timeout,
10924            "gate repair e2e",
10925            wait,
10926            |state, budget| async move {
10927                run_e2e_with_retry(
10928                    state,
10929                    &shell,
10930                    &commands,
10931                    &winner.worktree,
10932                    budget,
10933                    "gate repair e2e",
10934                )
10935                .await
10936            },
10937        )
10938        .await?;
10939        outcomes
10940    };
10941    let deferred = verify_inconclusive(&outcomes);
10942    let passed = outcomes.iter().all(CommandOutcome::ok)
10943        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10944    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10945        head,
10946        passed,
10947        outcomes,
10948    });
10949    state.save()?;
10950    if deferred {
10951        Ok(GateFix::Defer)
10952    } else if !passed {
10953        state.event(
10954            "gate",
10955            "verify.e2e failed after gate repair; nothing published",
10956        );
10957        Ok(GateFix::Stop)
10958    } else {
10959        Ok(GateFix::Retry)
10960    }
10961}
10962
10963/// Run the same configured gate for a landing head and a fresh graph.
10964pub(crate) async fn run_landing_gate(
10965    state: &mut RunState,
10966    winner: &Candidate,
10967    wait: &crate::repo_lock::WaitPolicy,
10968) -> Result<Vec<CommandOutcome>> {
10969    let shell = state.config.shell();
10970    let gate_commands = state.config.verify.gate.clone();
10971    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10972    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10973        Vec::new()
10974    } else {
10975        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10976        let cache_dir = state.config.cache_dir();
10977        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10978        let (outcomes, _) = with_cache_lease(
10979            state,
10980            cache_dir.as_deref(),
10981            "gate",
10982            "gate",
10983            &winner.worktree,
10984            &head,
10985            timeout,
10986            "final gate",
10987            wait,
10988            |state, budget| {
10989                let shell = shell.clone();
10990                let gate_commands = gate_commands.clone();
10991                let winner = winner.clone();
10992                async move {
10993                    Runner::run_pre_gate(state, &winner).await;
10994                    let (outcomes, timed_out_pids) = run_commands(
10995                        state,
10996                        "gate",
10997                        "gate",
10998                        0,
10999                        &shell,
11000                        &gate_commands,
11001                        &winner.worktree,
11002                        budget,
11003                    )
11004                    .await;
11005                    (outcomes, false, timed_out_pids)
11006                }
11007            },
11008        )
11009        .await?;
11010        outcomes
11011    };
11012    if outcomes.is_empty() {
11013        // Nothing configured to check — distinct from every other
11014        // silence in this run's event log, since an empty `gate` alone
11015        // no longer says whether the gate ran at all (see
11016        // `RunState::gate_ran`'s own doc).
11017        state.event("gate", "no gate commands configured; not tested");
11018    }
11019    for o in &outcomes {
11020        state.event(
11021            "gate",
11022            format!(
11023                "`{}` -> {}",
11024                o.command,
11025                if o.ok() {
11026                    "pass".to_owned()
11027                } else {
11028                    format!(
11029                        "FAIL ({:?})\n{}",
11030                        o.code,
11031                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11032                    )
11033                }
11034            ),
11035        );
11036    }
11037    Ok(outcomes)
11038}
11039
11040/// The pull request title to hand to `land::merge_subject`. A review-only run
11041/// opened by an earlier build titled its pull request with the review prompt;
11042/// that title is dropped (empty, so the fallback applies) rather than landed.
11043/// Any other title, including an operator's rename, passes through untouched,
11044/// and so does every title of a run that implements a task.
11045pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11046    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11047        ""
11048    } else {
11049        pr_title
11050    }
11051}
11052
11053/// What the squash subject falls back to when the pull request title is empty
11054/// or candidate-shaped: for a review-only run the derived title, never the
11055/// review prompt held in `instruction`.
11056pub fn landing_subject_source(state: &RunState) -> String {
11057    if is_review_run(state) {
11058        let winner = state.candidates.first().map_or('A', |c| c.label);
11059        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11060    }
11061    state.instruction.clone()
11062}
11063
11064/// `summary` without its `TITLE:` line, which the pull request title already
11065/// carries.
11066fn summary_without_title(summary: &str) -> String {
11067    let mut lines = summary.trim().lines().peekable();
11068    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11069        lines.next();
11070    }
11071    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11072}
11073
11074/// The pull request title and body for the winning candidate.
11075///
11076/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11077/// the task's own opening line via [`queue::title_from`] when there is none.
11078/// `state.instruction` can open with blank lines (`task_text` only rejects a
11079/// body that is blank *entirely*), which `title_from` skips.
11080///
11081/// Publishing uses an independent description generated from the final diff
11082/// and structured evidence. Internal summaries and reports remain local.
11083/// Non-publishing previews may display the original task and candidate summary.
11084/// The footer retains the run and candidate provenance.
11085#[cfg(test)]
11086fn pr_message(state: &RunState, winner: char) -> PrMessage {
11087    pr_message_with(state, winner, None)
11088}
11089
11090/// [`pr_message`] with what the branch of a review-only run says about itself.
11091/// `facts` is ignored for a run that implements a task.
11092#[cfg(test)]
11093fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11094    let raw = pr_message_raw(
11095        state,
11096        winner,
11097        facts,
11098        crate::pr_outcome::FinalHead::Unreported,
11099        ChangeNote::NotAsked,
11100    );
11101    let id = crate::scrub::Identity::current();
11102    PrMessage {
11103        title: crate::scrub::scrub(&raw.title, &id),
11104        body: crate::scrub::scrub(&raw.body, &id),
11105    }
11106}
11107
11108/// What the public description may say about the change itself.
11109#[derive(Debug, Clone, Copy)]
11110enum ChangeNote<'a> {
11111    /// A local preview or non-publishing merge mode: no public generation needed.
11112    NotAsked,
11113    /// A summary written from the final diff.
11114    Summary(&'a crate::run::ChangeSummary),
11115    /// A summary was asked for and could not be produced; the text says why.
11116    Unavailable(&'a str),
11117}
11118
11119fn pr_message_raw(
11120    state: &RunState,
11121    winner: char,
11122    facts: Option<&BranchFacts>,
11123    head: crate::pr_outcome::FinalHead<'_>,
11124    change: ChangeNote<'_>,
11125) -> PrMessage {
11126    if !matches!(change, ChangeNote::NotAsked) {
11127        let (title, mut body) = match change {
11128            ChangeNote::Summary(c) => (
11129                c.title.clone(),
11130                format!("## Summary\n\n{}\n\n", c.body.trim()),
11131            ),
11132            ChangeNote::Unavailable(why) => (
11133                neutral_title(state, winner),
11134                format!(
11135                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11136                ),
11137            ),
11138            ChangeNote::NotAsked => unreachable!(),
11139        };
11140        body.push_str(&public_pr_evidence(state, head));
11141        body.push_str(&format!(
11142            "\n---\nmagi:run/{} magi:candidate-{}\n",
11143            state.id,
11144            winner.to_ascii_lowercase()
11145        ));
11146        return PrMessage { title, body };
11147    }
11148    pr_preview(state, winner, facts, head)
11149}
11150
11151/// Append public facts from the run records after every prose generation,
11152/// including posting-gate rewrites. The agent cannot drop this provenance.
11153fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11154    let mut body = crate::pr_outcome::review_rounds(state);
11155    body.push_str(&crate::pr_outcome::final_verification(state, head));
11156    let open = state.last_round_findings();
11157    if !open.is_empty() {
11158        body.push_str("## Open review findings\n\n");
11159        for f in open {
11160            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11161        }
11162        body.push('\n');
11163    }
11164    if let Some(c) = state.change_summary.as_ref()
11165        && head == crate::pr_outcome::FinalHead::At(&c.head)
11166    {
11167        body.push_str(&format!(
11168            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11169            c.base, c.head
11170        ));
11171    }
11172    body
11173}
11174
11175/// Local-only preview of the operator's task and candidate summary.
11176fn pr_preview(
11177    state: &RunState,
11178    winner: char,
11179    facts: Option<&BranchFacts>,
11180    head: crate::pr_outcome::FinalHead<'_>,
11181) -> PrMessage {
11182    let summary = state
11183        .candidates
11184        .iter()
11185        .find(|c| c.label == winner)
11186        .map(|c| c.summary.as_str())
11187        .unwrap_or_default();
11188    // The fallback is the operator's own words and may not be English; GitHub
11189    // text always is, so a non-English task gets a neutral title instead.
11190    let review = is_review_run(state);
11191    let title = if review {
11192        facts
11193            .and_then(|f| {
11194                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11195            })
11196            .or_else(|| review_title(state))
11197            .or_else(|| {
11198                state
11199                    .candidates
11200                    .iter()
11201                    .find(|c| c.label == winner)
11202                    .filter(|c| !c.branch.starts_with("magi/"))
11203                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11204            })
11205            .unwrap_or_else(|| neutral_title(state, winner))
11206    } else {
11207        summary_title(summary).unwrap_or_else(|| {
11208            english_title(
11209                &state.instruction,
11210                PR_TITLE_MAX,
11211                state.config.graph.github_english(),
11212            )
11213            .unwrap_or_else(|| neutral_title(state, winner))
11214        })
11215    };
11216
11217    let mut body = String::new();
11218    let what = summary_without_title(summary);
11219    if !what.is_empty() {
11220        body.push_str("## Summary\n\n");
11221        body.push_str(&what);
11222        body.push_str("\n\n");
11223    }
11224
11225    // Review history uses structured facts across all rounds, never reports.
11226    body.push_str(&crate::pr_outcome::review_rounds(state));
11227    body.push_str(&crate::pr_outcome::final_verification(state, head));
11228    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11229
11230    let open = state.open_findings();
11231    if !open.is_empty() {
11232        body.push_str("## Open review findings\n\n");
11233        for f in &open {
11234            body.push_str(&format!(
11235                "- `{}` [{:?}] {}\n",
11236                f.id, f.severity, "Context retained in local review records."
11237            ));
11238        }
11239        body.push('\n');
11240    }
11241
11242    if let Some(fix) = fix
11243        && !fix.rejected.is_empty()
11244    {
11245        body.push_str("## Declined by the fixer\n\n");
11246        for r in &fix.rejected {
11247            body.push_str(&format!(
11248                "- `{}`: {}\n",
11249                r.id, "Reason retained in local review records."
11250            ));
11251        }
11252        body.push('\n');
11253    }
11254
11255    if review {
11256        // The review prompt is not the task; list what the branch carries.
11257        body.push_str("## Commits under review\n\n");
11258        if let Some(facts) = facts {
11259            let mut left = COMMITS_MAX_CHARS;
11260            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11261                let mut entry = format!("- {}\n", markdown_safe(subject));
11262                for l in markdown_safe(text).lines() {
11263                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11264                }
11265                if left == 0 {
11266                    body.push_str(&format!(
11267                        "- ... {} more commit(s)\n",
11268                        facts.commits.len() - i
11269                    ));
11270                    break;
11271                }
11272                if entry.len() > left {
11273                    // Even the first commit is cut: one huge body must not
11274                    // push the whole description past GitHub's limit.
11275                    let mut end = left;
11276                    while !entry.is_char_boundary(end) {
11277                        end -= 1;
11278                    }
11279                    entry.truncate(end);
11280                    entry.push_str("\n  ... (truncated)\n");
11281                    left = 0;
11282                } else {
11283                    left -= entry.len();
11284                }
11285                body.push_str(&entry);
11286            }
11287            if !facts.stat.trim().is_empty() {
11288                body.push_str(&format!(
11289                    "\n## Diff stat\n\n```\n{}\n```\n",
11290                    markdown_safe(facts.stat.trim())
11291                ));
11292            }
11293        } else {
11294            match &state.reviewed_commits {
11295                Some(subjects) => {
11296                    for s in subjects {
11297                        body.push_str(&format!("- {}\n", s.trim()));
11298                    }
11299                }
11300                None => {
11301                    // An older run kept only the prompt, with the commit list
11302                    // after its first paragraph.
11303                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11304                    body.push_str(rest.trim());
11305                    body.push('\n');
11306                }
11307            }
11308        }
11309    } else {
11310        let task = state.instruction.trim();
11311        let task = if task.is_empty() {
11312            "(empty task)"
11313        } else {
11314            task
11315        };
11316        body.push_str(&format!(
11317            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11318            task.replace("</details>", "&lt;/details&gt;")
11319        ));
11320    }
11321
11322    body.push_str(&format!(
11323        "\n---\nmagi:run/{} magi:candidate-{}\n",
11324        state.id,
11325        winner.to_ascii_lowercase()
11326    ));
11327
11328    PrMessage { title, body }
11329}
11330
11331/// The task with what the repository says about the existing work it names
11332/// appended, so an implementer knows what it started from and what it must
11333/// not redo. Unchanged when the task names nothing.
11334fn seeded_instruction(state: &RunState) -> String {
11335    match refs::describe(&state.seeds) {
11336        Some(facts) => format!(
11337            "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11338             Candidates start from the unmerged branch named above, when there \
11339             is one, and carry any unmerged commit named by sha as a \
11340             cherry-pick. Check that this is what the task meant before \
11341             building on it.",
11342            state.instruction
11343        ),
11344        None => state.instruction.clone(),
11345    }
11346}
11347
11348/// Does the winner have no commits ahead of the base it would land on?
11349/// Any failure to find out reads as "not empty": the merge then behaves as it
11350/// always did rather than refusing on a guess.
11351async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11352    let base = &state.base_branch;
11353    let mut against = base.clone();
11354    if mode == MergeMode::Pr {
11355        // A pull request lands on the remote's base, never the local branch
11356        // of the same name: if that cannot be read, "not empty" is the safe
11357        // answer.
11358        let remote = &state.config.merge.remote;
11359        let tracking = format!("{remote}/{base}");
11360        let fetched = git::fetch(repo, remote, base).await;
11361        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11362            against = tracking;
11363        } else {
11364            return false;
11365        }
11366    }
11367    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11368}
11369
11370/// Why nothing was opened for an empty winner, with what the task's own
11371/// references resolved to.
11372fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11373    let mut detail = format!(
11374        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11375         nothing to open a pull request for"
11376    );
11377    match refs::describe(&state.seeds) {
11378        Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11379        None => detail.push_str(
11380            "\nThe task names no existing branch or commit; if it means to land work \
11381             that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11382        ),
11383    }
11384    detail
11385}
11386
11387/// What the `Pr` merge does once it knows whether the branch already has an
11388/// open pull request.
11389#[derive(Debug, PartialEq, Eq)]
11390enum PrPlan {
11391    Create,
11392    Adopt { url: String, title: String },
11393    Stop(String),
11394}
11395
11396/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11397/// a failed lookup -> stop with the real reason. Never guesses.
11398fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11399    match found {
11400        Ok(land::OpenPr::None) => PrPlan::Create,
11401        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11402        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11403            "several open pull requests exist for this branch, not picking one: {}",
11404            urls.join(" ")
11405        )),
11406        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11407    }
11408}
11409
11410/// `gh pr create`, returning the PR url.
11411pub(crate) async fn gh_pr_create(
11412    cwd: &Path,
11413    base: &str,
11414    head: &str,
11415    title: &str,
11416    body: &str,
11417) -> Result<String> {
11418    let out = tokio::process::Command::new("gh")
11419        .args([
11420            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11421        ])
11422        .current_dir(cwd)
11423        .quiet()
11424        .stdin(std::process::Stdio::null())
11425        .output()
11426        .await
11427        .context("spawn gh")?;
11428    if out.status.success() {
11429        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11430    } else {
11431        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11432    }
11433}
11434
11435/// Never force deletion of uncommitted work, including unknown legacy trees.
11436async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11437    match std::fs::symlink_metadata(path) {
11438        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11439            // A previous cleanup may have removed the tree before a branch
11440            // deletion or state save failed. Unregister it if still present.
11441            git::worktree_remove_clean(repo, path).await?;
11442            return Ok(());
11443        }
11444        Err(e) => return Err(e.into()),
11445        Ok(_) => {}
11446    }
11447    if !git::is_clean(path).await? {
11448        bail!(
11449            "worktree has uncommitted work; refusing removal: {}",
11450            path.display()
11451        );
11452    }
11453    if !git::worktree_remove_clean(repo, path).await? {
11454        bail!("worktree removal refused: {}", path.display());
11455    }
11456    Ok(())
11457}
11458
11459/// Tear a run's worktrees and branches down.
11460///
11461/// `home` is where the updated `run.json` is saved (via
11462/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11463/// a housekeeping pass already has its own honest `home` handed to it, and
11464/// falling through to the global here would write back through whichever
11465/// directory some other process or test pinned into that `OnceLock` first,
11466/// not the one the caller actually resolved its `runs` and `state` from.
11467pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11468    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11469        bail!("refusing fold: {reason}");
11470    }
11471    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11472        bail!("candidate preservation incomplete; refusing cleanup");
11473    }
11474    // Serialize ordinary folding with the same starts/resumes as target-only
11475    // eviction. Otherwise a second janitor could remove the whole worktree
11476    // around a live output borrower even though target eviction was fenced.
11477    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11478    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11479    let _driver = crate::driver::maintenance(home, &state.id)?;
11480    let mut output_guards = Vec::new();
11481    for entry in std::fs::read_dir(state.worktree_root())
11482        .into_iter()
11483        .flatten()
11484        .flatten()
11485    {
11486        if entry.path().join(".git").is_file() {
11487            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11488        }
11489    }
11490    let repo = state.repo.clone();
11491    // CLI, Web and automatic cleanup all enter here, independently of
11492    // Runner::fold_losers. Protect deletion and shared config release too.
11493    let wait = repository_wait(state, None);
11494    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11495        Ok(lease) => lease,
11496        Err(e) => {
11497            state.save_under(home)?;
11498            return Err(e);
11499        }
11500    };
11501    // A merged PR proves only its published change. Preflight every source
11502    // before removing anything; a losing candidate or detached reviewer can
11503    // still retain independent commits or Git recovery checkpoints.
11504    let _source_outputs = if state.status == RunStatus::Merged
11505        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11506    {
11507        verify_merged_fold_sources(state, drop_winner, home).await?
11508    } else {
11509        Vec::new()
11510    };
11511    let root = state.worktree_root();
11512    let winner = state.tally.as_ref().map(|t| t.winner);
11513    let mut removed = Vec::new();
11514
11515    for i in 0..state.candidates.len() {
11516        let c = state.candidates[i].clone();
11517        let is_winner = Some(c.label) == winner;
11518        if is_winner && !drop_winner {
11519            continue;
11520        }
11521        let existed = c.worktree.exists();
11522        // A vanished directory can still have a Git registration that keeps
11523        // its branch checked out. Remove that registration before the branch.
11524        remove_preserved_worktree(&repo, &c.worktree).await?;
11525        if existed {
11526            removed.push(c.worktree.to_string_lossy().into_owned());
11527        }
11528        // A branch handed to a later run (and its pull request) is not this
11529        // run's to delete.
11530        let handed_over = state.released_branches.contains(&c.branch);
11531        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11532            if !git::branch_delete(&repo, &c.branch).await? {
11533                bail!("candidate branch removal refused");
11534            }
11535            removed.push(c.branch.clone());
11536        }
11537        state.candidates[i].folded = true;
11538    }
11539
11540    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11541        let path = name.path();
11542        let keep = !drop_winner
11543            && winner.is_some_and(|w| {
11544                path.file_name()
11545                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11546            });
11547        if keep {
11548            continue;
11549        }
11550        remove_preserved_worktree(&repo, &path).await?;
11551        removed.push(path.to_string_lossy().into_owned());
11552    }
11553
11554    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11555    // judge worktrees, so once the loop above has cleared all of them out,
11556    // the parent is a bare directory nobody else was ever going to remove -
11557    // git only ever managed what was inside it. Left alone, one of these
11558    // accumulates per fully-folded run; the operator's own machine had 74.
11559    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11560    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11561    // so does anything a run never claimed that happens to share the bay.
11562    remove_if_empty(&root);
11563
11564    if state.enabled_worktree_config && drop_winner {
11565        // A release, not a raw disable: some sibling run in this repository
11566        // may still hold its own reference (see `git::acquire_worktree_config`),
11567        // and only the last release actually turns the setting back off.
11568        git::release_worktree_config(&repo).await.ok();
11569        state.enabled_worktree_config = false;
11570    }
11571    state.save_under(home)?;
11572    Ok(removed)
11573}
11574
11575async fn verify_merged_fold_sources(
11576    state: &RunState,
11577    drop_winner: bool,
11578    home: &Path,
11579) -> Result<Vec<std::fs::File>> {
11580    let winner = state.tally.as_ref().map(|t| t.winner);
11581    let mut heads = BTreeSet::new();
11582    for candidate in &state.candidates {
11583        if Some(candidate.label) == winner && !drop_winner {
11584            continue;
11585        }
11586        match std::fs::symlink_metadata(&candidate.worktree) {
11587            Ok(_) => anyhow::ensure!(
11588                candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11589                "candidate recovery source is outside the inventoried run root: {}",
11590                candidate.worktree.display()
11591            ),
11592            Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11593            Err(error) => {
11594                return Err(error).context(format!(
11595                    "cannot inspect candidate recovery source: {}",
11596                    candidate.worktree.display()
11597                ));
11598            }
11599        }
11600        if git::branch_exists(&state.repo, &candidate.branch).await? {
11601            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11602            heads.extend(
11603                git::git(
11604                    &state.repo,
11605                    &["reflog", "show", "--format=%H", &candidate.branch],
11606                )
11607                .await?
11608                .lines()
11609                .map(str::to_owned),
11610            );
11611        }
11612    }
11613    let mut cargo_guards = Vec::new();
11614    let entries = match std::fs::read_dir(state.worktree_root()) {
11615        Ok(entries) => Some(entries),
11616        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11617        Err(error) => return Err(error.into()),
11618    };
11619    for entry in entries.into_iter().flatten() {
11620        let path = entry?.path();
11621        let metadata = std::fs::symlink_metadata(&path)?;
11622        anyhow::ensure!(
11623            metadata.is_dir() && !metadata.file_type().is_symlink(),
11624            "non-directory or symlink recovery source retained: {}",
11625            path.display()
11626        );
11627        if !drop_winner
11628            && winner.is_some_and(|w| {
11629                path.file_name()
11630                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11631            })
11632        {
11633            continue;
11634        }
11635        anyhow::ensure!(
11636            path.join(".git").is_file(),
11637            "unregistered recovery source retained: {}",
11638            path.display()
11639        );
11640        anyhow::ensure!(
11641            git::git(
11642                &path,
11643                &[
11644                    "status",
11645                    "--porcelain",
11646                    "--untracked-files=all",
11647                    "--ignored",
11648                    "--",
11649                    ".",
11650                    ":(exclude)target"
11651                ]
11652            )
11653            .await?
11654            .is_empty(),
11655            "source changes or ignored recovery files retained: {}",
11656            path.display()
11657        );
11658        match std::fs::symlink_metadata(path.join("target")) {
11659            Ok(_) => {
11660                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11661            }
11662            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11663            Err(error) => return Err(error.into()),
11664        }
11665        for checkpoint in [
11666            "rebase-merge",
11667            "rebase-apply",
11668            "sequencer",
11669            "CHERRY_PICK_HEAD",
11670            "MERGE_HEAD",
11671            "REBASE_HEAD",
11672            "REVERT_HEAD",
11673            "BISECT_LOG",
11674            "AUTO_MERGE",
11675            "MERGE_MSG",
11676        ] {
11677            let location = git::git(
11678                &path,
11679                &[
11680                    "rev-parse",
11681                    "--path-format=absolute",
11682                    "--git-path",
11683                    checkpoint,
11684                ],
11685            )
11686            .await?;
11687            anyhow::ensure!(
11688                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11689                if e.kind() == std::io::ErrorKind::NotFound),
11690                "Git recovery checkpoint retained: {checkpoint}"
11691            );
11692        }
11693        heads.insert(git::rev_parse(&path, "HEAD").await?);
11694        heads.extend(
11695            git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11696                .await?
11697                .lines()
11698                .map(str::to_owned),
11699        );
11700        if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11701            heads.insert(original);
11702        }
11703    }
11704    if !heads.is_empty() {
11705        // Never trust a run-wide merge label or the local checkout's branch.
11706        // Pin the fetched base once. A squash does not preserve commit IDs:
11707        // prove the winner's cumulative patch before accepting its ancestors.
11708        let reference = format!(
11709            "refs/remotes/{}/{}",
11710            state.config.merge.remote, state.base_branch
11711        );
11712        let base = git::rev_parse(&state.repo, &reference)
11713            .await
11714            .context("cannot prove merged sources against the fetched base")?;
11715        let mut squash_winner = None;
11716        let mut checked_squash = false;
11717        for head in heads {
11718            if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11719                .await?
11720                .ok()
11721            {
11722                continue;
11723            }
11724            if !checked_squash {
11725                squash_winner = proven_squash_winner(state, &base).await?;
11726                checked_squash = true;
11727            }
11728            let incorporated = if let Some(winner) = &squash_winner {
11729                git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11730                    .await?
11731                    .ok()
11732            } else {
11733                false
11734            };
11735            anyhow::ensure!(
11736                incorporated,
11737                "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11738            );
11739        }
11740    }
11741    Ok(cargo_guards)
11742}
11743
11744/// Called only for a confirmed merged run. Its label admits proof, but does
11745/// not prove that any particular local branch or recovery checkpoint landed.
11746/// Match the complete binary patch, including modes and blob IDs, against a
11747/// single-parent commit in the pinned fetched history. This works after later
11748/// base commits too, without trusting the current base tree to still match.
11749async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11750    let Some(winner) = state.winner() else {
11751        return Ok(None);
11752    };
11753    let head = git::rev_parse(&state.repo, &winner.branch).await?;
11754    if !git::git_raw(
11755        &state.repo,
11756        &["merge-base", "--is-ancestor", &state.base_commit, &head],
11757    )
11758    .await?
11759    .ok()
11760    {
11761        return Ok(None);
11762    }
11763    let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11764    if patch.is_empty() {
11765        return Ok(None);
11766    }
11767    let range = format!("{}..{base}", state.base_commit);
11768    let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11769    for commit in commits.lines() {
11770        let parent = format!("{commit}^");
11771        if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11772            return Ok(Some(head));
11773        }
11774    }
11775    Ok(None)
11776}
11777
11778async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11779    git::git(
11780        repo,
11781        &[
11782            "-c",
11783            "core.quotePath=true",
11784            "diff",
11785            "--binary",
11786            "--full-index",
11787            "--no-renames",
11788            "--no-ext-diff",
11789            "--no-textconv",
11790            "--no-color",
11791            "--ignore-submodules=none",
11792            "--submodule=short",
11793            from,
11794            to,
11795            "--",
11796        ],
11797    )
11798    .await
11799}
11800
11801/// Remove `dir` if it exists and has nothing in it.
11802///
11803/// Best-effort and silent by design: a directory that is not empty (a run
11804/// whose winner is still parked there, a stray file some other process left)
11805/// is exactly the case this must refuse, and a directory that is already gone
11806/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11807/// already refuses a non-empty directory, so the emptiness check below is
11808/// belt, not suspenders - it is what keeps this from ever attempting the
11809/// removal in the case that matters, rather than trusting `remove_dir`'s
11810/// error path to have no side effects if it ever changed.
11811fn remove_if_empty(dir: &Path) {
11812    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11813        std::fs::remove_dir(dir).ok();
11814    }
11815}
11816
11817/// Severity of the worst open finding in the last review round, for reporting.
11818pub fn worst_open(state: &RunState) -> Option<Severity> {
11819    state
11820        .reviews
11821        .last()?
11822        .reviews
11823        .iter()
11824        .flat_map(|r| r.findings.iter())
11825        .map(|f| f.severity)
11826        .max()
11827}
11828
11829/// `[worktree] setup` hides its products per worktree, which needs
11830/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11831/// worktrees, like the hook's; every place that runs setup calls this first.
11832async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11833    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11834        git::acquire_worktree_config(repo).await?;
11835        state.enabled_worktree_config = true;
11836    }
11837    Ok(())
11838}
11839
11840#[cfg(test)]
11841mod tests {
11842    #[tokio::test]
11843    async fn launcher_base_fetch_requires_remote_success() {
11844        let tmp = tempfile::tempdir().unwrap();
11845        let repo = tmp.path().join("repo");
11846        let remote = tmp.path().join("remote");
11847        std::fs::create_dir(&repo).unwrap();
11848        std::fs::create_dir(&remote).unwrap();
11849        git::git(&remote, &["init", "--bare", "-b", "main"])
11850            .await
11851            .unwrap();
11852        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11853        git::git(
11854            &repo,
11855            &[
11856                "-c",
11857                "user.name=Fixture",
11858                "-c",
11859                "user.email=fixture@example.invalid",
11860                "commit",
11861                "--allow-empty",
11862                "-m",
11863                "fixture",
11864            ],
11865        )
11866        .await
11867        .unwrap();
11868        git::git(
11869            &repo,
11870            &["remote", "add", "origin", remote.to_str().unwrap()],
11871        )
11872        .await
11873        .unwrap();
11874        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11875        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11876        assert_eq!(
11877            resolve_base(&repo, "main", "origin", None)
11878                .await
11879                .unwrap()
11880                .commit,
11881            head
11882        );
11883        #[cfg(unix)]
11884        {
11885            let policy: crate::config::Coordination = toml::from_str(
11886                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11887            ).unwrap();
11888            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11889            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11890            let held = crate::repo_lock::SeatLease::operation_with(
11891                &repo,
11892                Some(&policy),
11893                &crate::repo_lock::WaitPolicy::default(),
11894            )
11895            .await
11896            .unwrap();
11897            assert!(held.is_some());
11898            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11899            tokio::pin!(fetch);
11900            assert!(
11901                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11902                    .await
11903                    .is_err()
11904            );
11905            assert!(
11906                !repo.join(".git/FETCH_HEAD").exists(),
11907                "no fetch before admission"
11908            );
11909            drop(held);
11910            let fetched = fetch.await.unwrap();
11911            assert_eq!(fetched.commit, head);
11912            assert_eq!(fetched.remote, "origin");
11913            assert_eq!(fetched.branch, "main");
11914        }
11915        // A surviving tracking ref is not successful fetch evidence.
11916        git::git(
11917            &repo,
11918            &[
11919                "remote",
11920                "set-url",
11921                "origin",
11922                tmp.path().join("missing").to_str().unwrap(),
11923            ],
11924        )
11925        .await
11926        .unwrap();
11927        let error = resolve_base(&repo, "main", "origin", None)
11928            .await
11929            .unwrap_err()
11930            .to_string();
11931        assert!(error.contains("cannot read origin/main"));
11932        assert!(error.contains("refusing to branch off"));
11933    }
11934
11935    #[test]
11936    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11937        let leaked = vec!["chore(deps): update a crate".to_owned()];
11938        let own = "fix(daemon): apply a chosen action";
11939        assert!(should_retitle("", own, &leaked));
11940        assert!(should_retitle(
11941            &format!("{REVIEW_PROMPT_OPENING} `x`"),
11942            own,
11943            &leaked
11944        ));
11945        assert!(should_retitle(
11946            "chore: land candidate A of run 1",
11947            own,
11948            &leaked
11949        ));
11950        assert!(should_retitle(
11951            "magi: candidate A (uncommitted work)",
11952            own,
11953            &leaked
11954        ));
11955        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11956        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11957        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11958    }
11959
11960    #[test]
11961    fn pr_merge_plan_creates_adopts_or_stops() {
11962        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11963        assert_eq!(
11964            pr_merge_plan(Ok(land::OpenPr::One {
11965                url: "u".into(),
11966                title: "t".into()
11967            })),
11968            PrPlan::Adopt {
11969                url: "u".into(),
11970                title: "t".into()
11971            }
11972        );
11973        let PrPlan::Stop(many) =
11974            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11975        else {
11976            panic!("many must stop");
11977        };
11978        assert!(many.contains('a') && many.contains('b'));
11979        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11980            panic!("a failed lookup must stop");
11981        };
11982        assert!(err.contains("bad token"));
11983    }
11984
11985    use super::*;
11986    use crate::run::GateStatus;
11987    use std::collections::BTreeMap;
11988    use std::time::Duration;
11989
11990    fn conductor() -> AgentSpec {
11991        AgentSpec {
11992            id: "conductor".to_owned(),
11993            kind: crate::config::AgentKind::Command,
11994            model: None,
11995            command: vec!["true".to_owned()],
11996            extra_args: Vec::new(),
11997            env: BTreeMap::new(),
11998            prompt_delivery: None,
11999        }
12000    }
12001
12002    fn spec(id: &str) -> AgentSpec {
12003        AgentSpec {
12004            id: id.to_owned(),
12005            kind: crate::config::AgentKind::Command,
12006            model: None,
12007            command: vec!["true".to_owned()],
12008            extra_args: Vec::new(),
12009            env: BTreeMap::new(),
12010            prompt_delivery: None,
12011        }
12012    }
12013
12014    fn ids(xs: &[&str]) -> BTreeSet<String> {
12015        xs.iter().map(|s| (*s).to_owned()).collect()
12016    }
12017
12018    #[test]
12019    fn pick_successor_skips_an_agent_another_seat_holds() {
12020        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12021        let roster = [spec("a"), spec("b"), spec("c")];
12022        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12023        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12024    }
12025
12026    #[test]
12027    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12028        // The other seat started on c but was handed to d; c is free again.
12029        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12030        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12031        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12032        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12033        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12034    }
12035
12036    #[test]
12037    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12038        let roster = [spec("a"), spec("b")];
12039        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12040        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12041        let carried = ids(&["b"]);
12042        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12043        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12044    }
12045
12046    #[test]
12047    fn pick_successor_returns_none_without_any_untried_successor() {
12048        let roster = [spec("a"), spec("b")];
12049        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12050        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12051        let carried = ids(&["a"]);
12052        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12053    }
12054
12055    #[test]
12056    fn pick_successor_rescue_avoids_another_seats_occupant() {
12057        // b is a carried failure and free; a is held by the other seat.
12058        let roster = [spec("a"), spec("b"), spec("c")];
12059        let carried = ids(&["b", "c"]);
12060        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12061        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12062    }
12063
12064    #[test]
12065    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12066        let roster = [spec("a"), spec("b"), spec("c")];
12067        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12068        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12069    }
12070
12071    #[test]
12072    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12073        let roster = [spec("a"), spec("b"), spec("c")];
12074        let failed = ids(&["a", "b", "c"]);
12075        // Rescue looks at the whole roster, once per id, then runs out.
12076        let mut tried = ids(&["b"]);
12077        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12078        assert_eq!(first.id, "a");
12079        tried.insert(first.id.clone());
12080        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12081        assert_eq!(second.id, "c");
12082        tried.insert(second.id.clone());
12083        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12084    }
12085
12086    #[test]
12087    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12088        let roster = [spec("a"), spec("b")];
12089        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12090        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12091    }
12092
12093    #[test]
12094    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12095        let roster = [spec("a"), spec("b"), spec("c")];
12096        let h = SeatHistory {
12097            failed: ids(&["a"]),
12098            last_ok: Some("b".to_owned()),
12099            last_fail: None,
12100        };
12101        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12102        // A last answerer that left the roster, or later failed, is ignored.
12103        let gone = SeatHistory {
12104            last_ok: Some("zzz".to_owned()),
12105            ..h.clone()
12106        };
12107        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12108        let failed = SeatHistory {
12109            failed: ids(&["a", "b"]),
12110            last_ok: Some("b".to_owned()),
12111            last_fail: None,
12112        };
12113        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12114    }
12115
12116    #[test]
12117    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12118        let first = SeatState::new("review-1", "alpha", 7);
12119        let next = handover_seat("review-1", "gamma", 7);
12120        assert_ne!(first.claude_session, next.claude_session);
12121    }
12122
12123    #[test]
12124    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12125        let mut state = state_with_summary("x", "y");
12126        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12127        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12128        assert_ne!(a.claude_session, b.claude_session);
12129    }
12130
12131    #[test]
12132    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12133        let roster = [spec("a"), spec("b")];
12134        let h = SeatHistory {
12135            failed: ids(&["a", "b"]),
12136            ..SeatHistory::default()
12137        };
12138        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12139        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12140        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12141    }
12142
12143    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12144    // fallback loop depends on to terminate: it must walk forward from the
12145    // seat's own position, never restart at the front of the roster, and it
12146    // must never hand back an id already tried, however many times that id
12147    // happens to appear.
12148
12149    #[test]
12150    fn failure_signature_ignores_numbers_and_paths() {
12151        assert_eq!(
12152            failure_signature("exited with Some(2) and no usable output"),
12153            failure_signature("exited with Some(137) and no usable output")
12154        );
12155        assert_eq!(
12156            failure_signature("process exited with no usable output (exit code 2)"),
12157            failure_signature("process exited with no usable output (exit code 137)")
12158        );
12159        for (old, new) in [
12160            (
12161                "exited with Some(2) and no usable output",
12162                "process exited with no usable output (exit code 2)",
12163            ),
12164            (
12165                "exited with None and no usable output",
12166                "process exited with no usable output (exit code unavailable)",
12167            ),
12168        ] {
12169            let previous = FailClass::Other(failure_signature(old));
12170            let current = FailClass::Other(failure_signature(new));
12171            assert_eq!(previous, current, "resumption retains the failure class");
12172            assert!(!should_hand_over(Some(&previous), &current));
12173        }
12174        assert_eq!(
12175            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12176            failure_signature("cannot open /var/x.txt: denied")
12177        );
12178        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12179    }
12180
12181    #[test]
12182    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12183        let other = FailClass::Other("x".into());
12184        assert!(should_hand_over(None, &FailClass::Quota));
12185        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12186        assert!(should_hand_over(
12187            Some(&FailClass::Timeout),
12188            &FailClass::Timeout
12189        ));
12190        assert!(should_hand_over(None, &other));
12191        assert!(!should_hand_over(Some(&other), &other));
12192        assert!(should_hand_over(
12193            Some(&other),
12194            &FailClass::Other("y".into())
12195        ));
12196        // A quota or timeout in between ends the run of identical failures.
12197        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12198        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12199    }
12200
12201    #[test]
12202    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12203        let a = SeatState::new("judge-1", "alpha", 7);
12204        let b = handover_seat("judge-1", "beta", 7);
12205        assert_ne!(a.claude_session, b.claude_session);
12206        assert_eq!(b.turns, 0);
12207    }
12208
12209    #[test]
12210    fn a_timeout_is_classified_apart_from_other_failures() {
12211        assert_eq!(
12212            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12213            Some(FailClass::Timeout)
12214        );
12215        assert!(matches!(
12216            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12217            Some(FailClass::Other(_))
12218        ));
12219    }
12220
12221    #[test]
12222    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12223        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12224        let tried = BTreeSet::from(["beta".to_owned()]);
12225        // beta sits at index 1; the next candidate is gamma, never alpha —
12226        // which is very likely a different candidate slot's own agent.
12227        let next = next_untried_in_roster(&roster, 1, &tried);
12228        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12229    }
12230
12231    #[test]
12232    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12233        let roster = vec![spec("alpha"), spec("beta")];
12234        let tried = BTreeSet::from(["beta".to_owned()]);
12235        // beta is the roster's last entry: nothing follows it, and alpha —
12236        // earlier in the roster, almost certainly a different candidate
12237        // slot's own agent — must not be reached by wrapping back to it.
12238        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12239    }
12240
12241    #[test]
12242    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12243        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12244        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12245        // beta (index 1) and gamma (index 2, the only entry after it) have
12246        // both been tried; alpha (index 0) never has, but it comes before
12247        // beta's own position, so there is nothing further for this seat.
12248        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12249    }
12250
12251    #[test]
12252    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12253        let roster = vec![spec("a"), spec("a"), spec("b")];
12254        let tried = BTreeSet::from(["a".to_owned()]);
12255        let next = next_untried_in_roster(&roster, 0, &tried);
12256        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12257    }
12258
12259    #[test]
12260    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12261        let roster = vec![spec("a"), spec("b")];
12262        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12263        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12264    }
12265
12266    #[test]
12267    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12268        let dir = tempfile::tempdir().unwrap();
12269        let bay = dir.path().join("ffff");
12270
12271        // Not there yet: nothing to do, nothing to panic on.
12272        remove_if_empty(&bay);
12273        assert!(!bay.exists());
12274
12275        // Something still inside - the winner's worktree, or a stray file -
12276        // keeps the directory standing.
12277        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12278        remove_if_empty(&bay);
12279        assert!(bay.exists(), "non-empty directory must survive");
12280
12281        // Once the last entry is gone, so is the directory itself.
12282        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12283        remove_if_empty(&bay);
12284        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12285    }
12286
12287    // `round_is_clean` is the exact decision this task fixed: a round with a
12288    // seat that never answered must not read the same as a round every seat
12289    // actually reviewed. These are deterministic and process-free by design —
12290    // the equivalent end-to-end check (a real reviewer timing out under a
12291    // live graph run) is a genuine race against wall-clock contention, and a
12292    // spawn slow enough to blow even a generous budget under a loaded test
12293    // run must not turn this specific regression check flaky.
12294
12295    #[test]
12296    fn a_full_panel_that_found_nothing_is_clean() {
12297        assert!(round_is_clean(
12298            0,
12299            true,
12300            2,
12301            2,
12302            0,
12303            IncompleteReviewPolicy::Block
12304        ));
12305    }
12306
12307    #[test]
12308    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12309        assert!(!round_is_clean(
12310            0,
12311            true,
12312            1,
12313            2,
12314            0,
12315            IncompleteReviewPolicy::Block
12316        ));
12317    }
12318
12319    #[test]
12320    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12321        assert!(!round_is_clean(
12322            1,
12323            true,
12324            1,
12325            2,
12326            0,
12327            IncompleteReviewPolicy::Warn
12328        ));
12329    }
12330
12331    #[test]
12332    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12333        assert!(round_is_clean(
12334            0,
12335            true,
12336            1,
12337            2,
12338            0,
12339            IncompleteReviewPolicy::Warn
12340        ));
12341    }
12342
12343    #[test]
12344    fn a_full_panel_with_an_open_finding_is_not_clean() {
12345        assert!(!round_is_clean(
12346            1,
12347            true,
12348            2,
12349            2,
12350            0,
12351            IncompleteReviewPolicy::Block
12352        ));
12353    }
12354
12355    #[test]
12356    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12357        assert!(!round_is_clean(
12358            0,
12359            false,
12360            2,
12361            2,
12362            0,
12363            IncompleteReviewPolicy::Block
12364        ));
12365    }
12366
12367    // The stall this task closes: under the default `block` policy, a seat
12368    // missing only because it was rate limited must not force a wait for a
12369    // session limit that will not lift by the next round. `round_is_clean`
12370    // is where that quorum carve-out lives; the review loop around it never
12371    // changes what a reviewer's vote or a finding's severity means.
12372
12373    #[test]
12374    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12375        // 1 of 2 answered, and the one missing was quota'd — the exact
12376        // "review-2 rate limited (quota)" shape from the field report.
12377        assert!(round_is_clean(
12378            0,
12379            true,
12380            1,
12381            2,
12382            1,
12383            IncompleteReviewPolicy::Block
12384        ));
12385    }
12386
12387    #[test]
12388    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12389        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12390        // not a quota loss (`quota_missing` stays 0) — worth another try.
12391        assert!(!round_is_clean(
12392            0,
12393            true,
12394            1,
12395            2,
12396            0,
12397            IncompleteReviewPolicy::Block
12398        ));
12399    }
12400
12401    #[test]
12402    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12403        assert!(!round_is_clean(
12404            1,
12405            true,
12406            1,
12407            2,
12408            1,
12409            IncompleteReviewPolicy::Block
12410        ));
12411        assert!(!round_is_clean(
12412            0,
12413            false,
12414            1,
12415            2,
12416            1,
12417            IncompleteReviewPolicy::Block
12418        ));
12419    }
12420
12421    #[test]
12422    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12423        // Every seat quota'd, nobody answered: there is no panel to decide
12424        // on, so this must fall through to the existing block-and-retry
12425        // fallback rather than call an unreviewed patch clean.
12426        assert!(!round_is_clean(
12427            0,
12428            true,
12429            0,
12430            2,
12431            2,
12432            IncompleteReviewPolicy::Block
12433        ));
12434    }
12435
12436    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12437        CommandOutcome {
12438            command: "test".to_owned(),
12439            code,
12440            output_tail: String::new(),
12441            duration_ms: 0,
12442            resource_blocked,
12443        }
12444    }
12445
12446    #[test]
12447    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12448        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12449        assert!(
12450            !verify_inconclusive(&[outcome(Some(1), false)]),
12451            "an ordinary failure is still evidence about the patch"
12452        );
12453        assert!(verify_inconclusive(&[outcome(None, true)]));
12454        assert!(
12455            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12456            "one inconclusive outcome taints the whole batch"
12457        );
12458        assert!(!verify_inconclusive(&[]));
12459    }
12460
12461    #[tokio::test]
12462    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12463        // Alive for the first two checks, then dead - confirms the loop
12464        // actually re-polls rather than deciding once and sleeping out the
12465        // ceiling regardless.
12466        let calls = std::sync::atomic::AtomicUsize::new(0);
12467        let started = Instant::now();
12468        wait_for_pids_with(
12469            &[123],
12470            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12471            Duration::from_millis(5),
12472            Duration::from_secs(5),
12473        )
12474        .await;
12475        assert!(
12476            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12477            "must keep checking rather than deciding on the first answer"
12478        );
12479        assert!(
12480            started.elapsed() < Duration::from_secs(1),
12481            "must return the moment it is confirmed dead, not wait out the ceiling"
12482        );
12483    }
12484
12485    #[tokio::test]
12486    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12487        let started = Instant::now();
12488        wait_for_pids_with(
12489            &[123],
12490            |_| true, // never reports dead
12491            Duration::from_millis(5),
12492            Duration::from_millis(30),
12493        )
12494        .await;
12495        let elapsed = started.elapsed();
12496        assert!(
12497            elapsed >= Duration::from_millis(30),
12498            "must not give up before its own ceiling: {elapsed:?}"
12499        );
12500        assert!(
12501            elapsed < Duration::from_secs(1),
12502            "must not wait past its own ceiling either: {elapsed:?}"
12503        );
12504    }
12505
12506    #[tokio::test]
12507    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12508        let started = Instant::now();
12509        wait_for_pids_with(
12510            &[],
12511            |_| true,
12512            Duration::from_secs(5),
12513            Duration::from_secs(5),
12514        )
12515        .await;
12516        assert!(
12517            started.elapsed() < Duration::from_millis(200),
12518            "an empty pid list has nothing to confirm"
12519        );
12520    }
12521
12522    // `review_conclusion` is the exact decision the review hand-off task
12523    // fixed: a round budget spent (or a tree that stopped moving) must not
12524    // collapse into `Blocked` regardless of what verification actually
12525    // said. Deterministic and process-free for the same reason the
12526    // `round_is_clean` family above is.
12527    fn review_round(
12528        clean: bool,
12529        blocking: usize,
12530        answered: usize,
12531        expected: usize,
12532        progressed: bool,
12533        e2e_ok: bool,
12534    ) -> ReviewRound {
12535        ReviewRound {
12536            round: 1,
12537            head: "h".to_owned(),
12538            verified_head: None,
12539            verified_at: None,
12540            reviews: Vec::new(),
12541            e2e: vec![CommandOutcome {
12542                command: "test".to_owned(),
12543                code: Some(if e2e_ok { 0 } else { 1 }),
12544                output_tail: String::new(),
12545                duration_ms: 0,
12546                resource_blocked: false,
12547            }],
12548            verify_retried: false,
12549            e2e_deferred: false,
12550            e2e_defer_reason: None,
12551            fix: None,
12552            blocking,
12553            answered,
12554            expected,
12555            clean,
12556            progressed,
12557            vote_split: false,
12558            reconsideration: Vec::new(),
12559            verdict: None,
12560        }
12561    }
12562
12563    #[test]
12564    fn review_conclusion_is_none_when_nothing_has_run() {
12565        assert_eq!(review_conclusion(&[], 3), None);
12566    }
12567
12568    #[test]
12569    fn review_conclusion_is_none_while_rounds_remain() {
12570        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12571        assert_eq!(review_conclusion(&rounds, 3), None);
12572    }
12573
12574    #[test]
12575    fn review_conclusion_is_gating_once_a_round_is_clean() {
12576        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12577        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12578    }
12579
12580    #[test]
12581    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12582        let rounds = vec![
12583            review_round(false, 1, 2, 2, true, true),
12584            review_round(false, 1, 2, 2, true, true),
12585        ];
12586        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12587    }
12588
12589    #[test]
12590    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12591        let rounds = vec![
12592            review_round(false, 1, 2, 2, true, true),
12593            review_round(false, 1, 2, 2, true, false),
12594        ];
12595        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12596    }
12597
12598    #[test]
12599    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12600        // Magi never got a command to run against this round's own head — a
12601        // resource-blocked attempt, not a red one — so this must never
12602        // settle on `Blocked` the way a genuine e2e failure would. `None`
12603        // here is what tells `Runner::review_loop` to retry the check
12604        // itself rather than trust this cheap recomputation with a verdict
12605        // it cannot actually produce.
12606        let mut blocked = review_round(false, 1, 2, 2, true, false);
12607        blocked.e2e[0].resource_blocked = true;
12608        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12609        assert_eq!(review_conclusion(&rounds, 2), None);
12610    }
12611
12612    #[test]
12613    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12614        // Missing input, not a verified tree — never a hand-off candidate.
12615        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12616        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12617    }
12618
12619    #[test]
12620    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12621        let rounds = vec![
12622            review_round(false, 1, 2, 2, false, true),
12623            review_round(false, 1, 2, 2, false, true),
12624        ];
12625        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12626    }
12627
12628    fn secs(n: u64) -> Duration {
12629        Duration::from_secs(n)
12630    }
12631
12632    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12633    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12634    #[tokio::test]
12635    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12636        ask_test_home();
12637        let tmp = tempfile::tempdir().unwrap();
12638        let repo = tmp.path().join("repo");
12639        std::fs::create_dir_all(&repo).unwrap();
12640        init_repo(&repo);
12641        let g = |args: &[&str]| {
12642            let out = std::process::Command::new("git")
12643                .args(args)
12644                .current_dir(&repo)
12645                .quiet()
12646                .output()
12647                .unwrap();
12648            assert!(
12649                out.status.success(),
12650                "{}",
12651                String::from_utf8_lossy(&out.stderr)
12652            );
12653        };
12654        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12655        g(&["checkout", "-b", "historical"]);
12656        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12657        g(&["add", "."]);
12658        g(&["commit", "-m", "old alternative"]);
12659        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12660        g(&["checkout", "-b", "magi/task-selected", &base]);
12661        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12662        g(&["add", "."]);
12663        g(&["commit", "-m", "measurement infrastructure"]);
12664        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12665        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12666        g(&["add", "."]);
12667        g(&["commit", "-m", "dependent implementation"]);
12668        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12669        g(&["checkout", "main"]);
12670        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12671        g(&["add", "."]);
12672        g(&["commit", "-m", "unrelated base change"]);
12673        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12674        let origin = tmp.path().join("origin.git");
12675        g(&[
12676            "clone",
12677            "--bare",
12678            repo.to_str().unwrap(),
12679            origin.to_str().unwrap(),
12680        ]);
12681        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12682        g(&["fetch", "origin"]);
12683        let selection = crate::pr_recovery::Selection {
12684            url: "https://example.test/project/pull/1".into(),
12685            branch: "magi/task-selected".into(),
12686            head: head.clone(),
12687            source_head: None,
12688            source_run: None,
12689            source_proof: None,
12690            disposition: Some(crate::pr_recovery::Disposition::Review),
12691            instruction: "preserve measured behavior".into(),
12692            approved_at: jiff::Timestamp::now(),
12693        };
12694        let mut template = runner_at(RunStatus::Reviewing);
12695        template.state.repo = repo.clone();
12696        template.state.base_commit = current.clone();
12697        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12698        template.state.origin = Some(crate::run::Origin::queue("selected"));
12699        template.state.pr_head = Some(selection.branch.clone());
12700        template.state.pr_recovery = Some(selection.clone());
12701        let mut runner =
12702            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12703                .await
12704                .unwrap();
12705        let wt = runner.state.winner().unwrap().worktree.clone();
12706        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12707        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12708        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12709        assert!(runner.state.seeds.is_empty());
12710        assert_eq!(runner.state.instruction, "preserve measured behavior");
12711        runner.sync_to_base().await.unwrap();
12712        assert_eq!(
12713            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12714            "prerequisite"
12715        );
12716        assert_eq!(
12717            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12718            "measured implementation"
12719        );
12720        assert_eq!(
12721            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12722            "current main change"
12723        );
12724        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12725        assert_eq!(
12726            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12727            head
12728        );
12729
12730        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12731        let mut partial = runner_at(RunStatus::Reviewing);
12732        partial.state.repo = repo.clone();
12733        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12734        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12735            run: Some("exit 1".into()),
12736            ..Default::default()
12737        }];
12738        let selection = runner.state.pr_recovery.clone().unwrap();
12739        partial.state.pr_recovery = Some(selection.clone());
12740        partial.state.pr_head = Some(selection.branch.clone());
12741        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12742        let id = partial.state.id.clone();
12743        let error = match Runner::open_selected(
12744            &repo,
12745            partial.state,
12746            partial.roles,
12747            1,
12748            runner.state.base_commit.clone(),
12749            selection,
12750        )
12751        .await
12752        {
12753            Ok(_) => panic!("setup must fail"),
12754            Err(e) => e,
12755        };
12756        assert_eq!(
12757            error
12758                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12759                .unwrap()
12760                .run,
12761            id
12762        );
12763        let failed = RunState::load(&id).unwrap();
12764        assert_eq!(failed.status, RunStatus::Blocked);
12765        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12766        assert!(Runner::resume(&id).is_err());
12767        runner.state.status = RunStatus::Ready;
12768        runner.state.driver_exited = true;
12769        runner.state.save().unwrap();
12770        let mut retry = runner_at(RunStatus::Reviewing);
12771        retry.state.repo = repo.clone();
12772        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12773        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12774        reselected.approved_at = jiff::Timestamp::now();
12775        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12776        retry.state.pr_head = Some(reselected.branch.clone());
12777        retry.state.pr_recovery = Some(reselected.clone());
12778        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12779            .await
12780            .unwrap();
12781        let restarted = Runner::open_selected(
12782            &repo,
12783            retry.state,
12784            retry.roles,
12785            1,
12786            runner.state.base_commit.clone(),
12787            reselected,
12788        )
12789        .await
12790        .unwrap();
12791        assert_eq!(
12792            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12793                .await
12794                .unwrap(),
12795            head
12796        );
12797        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12798        assert_eq!(
12799            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12800            head
12801        );
12802    }
12803
12804    #[tokio::test]
12805    #[cfg(unix)]
12806    async fn fallback_and_interrupted_restoration_require_coordination() {
12807        use std::os::fd::AsRawFd as _;
12808        ask_test_home();
12809        let tmp = tempfile::tempdir().unwrap();
12810        let mut runner = registered_runner(tmp.path()).await;
12811        runner.state.config.graph.timeout_resource_wait = 1;
12812        let winner = runner.state.candidates[0].clone();
12813        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12814        let leftover = winner.worktree.join("leftover");
12815        std::fs::write(&leftover, "untrusted edits").unwrap();
12816        let next = tmp.path().join("next-fixer");
12817        let mut failed = spec("first");
12818        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12819        let mut successor = spec("second");
12820        successor.command = vec![
12821            "sh".into(),
12822            "-c".into(),
12823            format!("touch '{}'; echo done", next.display()),
12824        ];
12825        let attempts = vec![
12826            (failed.clone(), "first".into()),
12827            (successor.clone(), "second".into()),
12828        ];
12829        let lock_path = runner.state.repo.join(".git/updater.lock");
12830        let external = std::fs::File::create(&lock_path).unwrap();
12831        assert_eq!(
12832            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12833            0
12834        );
12835        let artifacts = tmp.path().join("artifacts");
12836        let build = |spec, seat| SeatJob {
12837            spec,
12838            seat,
12839            cwd: winner.worktree.clone(),
12840            prompt: "fixture".into(),
12841            timeout: Duration::from_secs(10),
12842            allow_write: true,
12843            sessions: false,
12844            artifacts: artifacts.clone(),
12845            stem: "fix".into(),
12846            handover: None,
12847        };
12848        let (_, _, out) = runner
12849            .ask_fixer_among(
12850                attempts.clone(),
12851                "graft",
12852                None,
12853                Some((&winner.worktree, &head)),
12854                build,
12855            )
12856            .await;
12857        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12858        assert!(leftover.exists(), "contention must prevent clean");
12859        assert!(!next.exists());
12860        assert!(runner.state.handovers.is_empty());
12861        // Early interrupted-repair rollback uses the same protected route.
12862        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12863            from_run: "earlier".into(),
12864            generation: 1,
12865            mode: crate::recovery::Mode::VerifyRepair,
12866            reasons: Vec::new(),
12867            reviewed_head: None,
12868            rounds: Vec::new(),
12869            open_findings: Vec::new(),
12870            evidence: None,
12871            criteria_inherited: true,
12872            repair: crate::recovery::Repair::Started {
12873                before: head.clone(),
12874            },
12875        });
12876        assert!(
12877            runner
12878                .recovery_repair(&winner)
12879                .await
12880                .unwrap_err()
12881                .is::<crate::repo_lock::ResourceBlocked>()
12882        );
12883        assert!(matches!(
12884            runner.state.recovery.as_ref().unwrap().repair,
12885            crate::recovery::Repair::Started { .. }
12886        ));
12887        assert!(leftover.exists());
12888        drop(external);
12889        runner.recovery_repair(&winner).await.unwrap();
12890        assert!(!leftover.exists());
12891        assert!(matches!(
12892            runner.state.recovery.as_ref().unwrap().repair,
12893            crate::recovery::Repair::Done { .. }
12894        ));
12895        // A hard lock-open failure also leaves HEAD and untracked work intact.
12896        std::fs::remove_file(&lock_path).unwrap();
12897        std::fs::create_dir(&lock_path).unwrap();
12898        std::fs::write(&leftover, "untrusted edits").unwrap();
12899        let (_, _, out) = runner
12900            .ask_fixer_among(
12901                attempts,
12902                "graft",
12903                None,
12904                Some((&winner.worktree, &head)),
12905                build,
12906            )
12907            .await;
12908        assert!(matches!(out, AgentOutcome::Failed(_)));
12909        assert!(leftover.exists());
12910        assert!(!next.exists());
12911        assert_eq!(
12912            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12913            head
12914        );
12915    }
12916
12917    #[cfg(unix)]
12918    async fn registered_runner(root: &Path) -> Runner {
12919        ask_test_home();
12920        let repo = root.join("repo");
12921        std::fs::create_dir_all(&repo).unwrap();
12922        init_repo(&repo);
12923        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12924        let mut runner = runner_at(RunStatus::Prep);
12925        runner.state.repo = repo.clone();
12926        runner.state.config.coordination = crate::config::test_coordination();
12927        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12928        runner.state.config.graph.worktree_root = Some(root.join("wt"));
12929        runner.state.config.graph.advise = false;
12930        runner.roles.implementers = vec![spec("implement")];
12931        runner.roles.judges = vec![spec("judge")];
12932        runner.prep().await.unwrap();
12933        runner
12934    }
12935
12936    #[tokio::test]
12937    #[cfg(unix)]
12938    async fn parked_implementer_persists_completed_work() {
12939        for registered in [false, true] {
12940            let tmp = tempfile::tempdir().unwrap();
12941            let mut runner = registered_runner(tmp.path()).await;
12942            if !registered {
12943                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12944            }
12945            let marker = tmp.path().join("started");
12946            let release = tmp.path().join("release");
12947            let mut implementer = spec("implement");
12948            implementer.command = vec![
12949                "sh".into(),
12950                "-c".into(),
12951                format!(
12952                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12953                    marker.display(),
12954                    release.display()
12955                ),
12956            ];
12957            runner.roles.implementers = vec![implementer.clone()];
12958            runner.state.config.agents = vec![implementer];
12959            let pause = runner.pause.clone();
12960            let (result, ()) = tokio::join!(runner.implement(), async {
12961                tokio::time::timeout(Duration::from_secs(10), async {
12962                    while !marker.exists() {
12963                        tokio::time::sleep(Duration::from_millis(10)).await;
12964                    }
12965                })
12966                .await
12967                .unwrap();
12968                pause.park();
12969                std::fs::write(&release, "go").unwrap();
12970            });
12971            result.unwrap();
12972            let candidate = &runner.state.candidates[0];
12973            assert!(candidate.commits > 0, "{candidate:?}");
12974            assert!(candidate.summary.contains("saved-result"));
12975            assert_eq!(candidate.files, 1);
12976            let before = candidate.commits;
12977            runner.implement().await.unwrap();
12978            assert_eq!(runner.state.candidates[0].commits, before);
12979        }
12980    }
12981
12982    #[tokio::test]
12983    #[cfg(unix)]
12984    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12985        let tmp = tempfile::tempdir().unwrap();
12986        let mut runner = registered_runner(tmp.path()).await;
12987        let winner = runner.state.candidates[0].clone();
12988        let cache = tmp.path().join("cache");
12989        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12990        runner.state.config.graph.timeout_resource_wait = 0;
12991        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12992        let crate::cache::AcquireOutcome::Acquired(held) =
12993            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12994        else {
12995            panic!("cache must be free")
12996        };
12997        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12998            from_run: "earlier".into(),
12999            generation: 1,
13000            mode: crate::recovery::Mode::VerifyRepair,
13001            reasons: Vec::new(),
13002            reviewed_head: None,
13003            rounds: Vec::new(),
13004            open_findings: Vec::new(),
13005            evidence: None,
13006            criteria_inherited: true,
13007            repair: crate::recovery::Repair::NotStarted,
13008        });
13009        runner.recovery_repair(&winner).await.unwrap();
13010        assert!(matches!(
13011            runner.state.recovery.as_ref().unwrap().repair,
13012            crate::recovery::Repair::WaitingForRestore { .. }
13013        ));
13014        drop(held);
13015        runner.state.resource_block = None;
13016        runner.state.config.graph.timeout_resource_wait = 1;
13017        runner.recovery_repair(&winner).await.unwrap();
13018        assert!(matches!(
13019            runner.state.recovery.as_ref().unwrap().repair,
13020            crate::recovery::Repair::Done { .. }
13021        ));
13022        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13023    }
13024
13025    #[tokio::test]
13026    #[cfg(unix)]
13027    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13028        use std::os::fd::AsRawFd as _;
13029        for graft in [false, true] {
13030            let tmp = tempfile::tempdir().unwrap();
13031            let mut runner = registered_runner(tmp.path()).await;
13032            runner.state.config.graph.timeout_resource_wait = 1;
13033            let winner = runner.state.candidates[0].clone();
13034            let marker = tmp.path().join("fixer-started");
13035            let mut fixer = spec("fixer");
13036            fixer.command = vec![
13037                "sh".into(),
13038                "-c".into(),
13039                format!("echo started >> '{}'; echo done", marker.display()),
13040            ];
13041            runner.roles.fixer = Some(vec![fixer]);
13042            if graft {
13043                let mut loser = winner.clone();
13044                loser.label = 'B';
13045                loser.index = 1;
13046                runner.state.candidates.push(loser);
13047                runner.state.config.graph.graft_runner_up = true;
13048                runner.state.tally = Some(Tally {
13049                    first_choice: BTreeMap::new(),
13050                    borda: BTreeMap::new(),
13051                    winner: winner.label,
13052                    rankings: 1,
13053                    unanimous_initial: true,
13054                    deliberated: false,
13055                    changed_votes: 0,
13056                    unanimous_final: true,
13057                    tie_break: None,
13058                    judges: 1,
13059                    present: 1,
13060                    quorum: 1,
13061                    met_quorum: true,
13062                    uncontested: None,
13063                });
13064                runner.state.judgements.push(crate::run::Judgement {
13065                    judge: 1,
13066                    seat: "judge-1".into(),
13067                    agent: "judge".into(),
13068                    ranking: vec![winner.label, 'B'],
13069                    reasons: BTreeMap::new(),
13070                    confidence: None,
13071                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13072                    order: vec![0, 1],
13073                    failed: None,
13074                    duration_ms: 0,
13075                });
13076            } else {
13077                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13078                    from_run: "earlier".into(),
13079                    generation: 1,
13080                    mode: crate::recovery::Mode::VerifyRepair,
13081                    reasons: Vec::new(),
13082                    reviewed_head: None,
13083                    rounds: Vec::new(),
13084                    open_findings: Vec::new(),
13085                    evidence: None,
13086                    criteria_inherited: true,
13087                    repair: crate::recovery::Repair::NotStarted,
13088                });
13089            }
13090            let reader = crate::repo_lock::SeatLease::for_checkout(
13091                &winner.worktree,
13092                false,
13093                crate::config::test_coordination().as_ref(),
13094            )
13095            .await
13096            .unwrap();
13097            let pause = runner.pause.clone();
13098            let (result, ()) = tokio::join!(
13099                async {
13100                    if graft {
13101                        runner.graft().await
13102                    } else {
13103                        runner.recovery_repair(&winner).await
13104                    }
13105                },
13106                async {
13107                    tokio::time::sleep(Duration::from_millis(100)).await;
13108                    pause.park();
13109                }
13110            );
13111            result.unwrap();
13112            assert!(!marker.exists(), "a blocked seat must not spawn");
13113            // Exercise the persisted state, as a binary replacement/resume does.
13114            runner.state = RunState::load(&runner.state.id).unwrap();
13115            if graft {
13116                assert_eq!(
13117                    runner.state.graft.as_ref().unwrap().outcome,
13118                    GraftOutcome::WaitingForRestore
13119                );
13120            } else {
13121                assert!(matches!(
13122                    runner.state.recovery.as_ref().unwrap().repair,
13123                    crate::recovery::Repair::WaitingForRestore { .. }
13124                ));
13125            }
13126            drop(reader);
13127            runner.pause = Pause::new();
13128            runner.state.resource_block = None;
13129            // Deferred restoration must still respect an external lock owner.
13130            let external =
13131                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13132            assert_eq!(
13133                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13134                0
13135            );
13136            let leftover = winner.worktree.join("leftover");
13137            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13138            let result = if graft {
13139                runner.graft().await
13140            } else {
13141                runner.recovery_repair(&winner).await
13142            };
13143            assert!(
13144                result
13145                    .unwrap_err()
13146                    .is::<crate::repo_lock::ResourceBlocked>()
13147            );
13148            assert!(leftover.exists());
13149            assert!(!marker.exists());
13150            drop(external);
13151            runner.state.resource_block = None;
13152            let result = if graft {
13153                runner.graft().await
13154            } else {
13155                runner.recovery_repair(&winner).await
13156            };
13157            result.unwrap();
13158            assert!(!leftover.exists(), "restore must precede retry");
13159            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13160            if graft {
13161                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13162            } else {
13163                assert!(matches!(
13164                    runner.state.recovery.as_ref().unwrap().repair,
13165                    crate::recovery::Repair::Done { .. }
13166                ));
13167            }
13168        }
13169    }
13170
13171    #[tokio::test]
13172    #[cfg(unix)]
13173    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13174        use std::os::fd::AsRawFd as _;
13175        let tmp = tempfile::tempdir().unwrap();
13176        let mut runner = registered_runner(tmp.path()).await;
13177        runner.state.config.graph.timeout_resource_wait = 1;
13178        let winner = runner.state.candidates[0].clone();
13179        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13180        runner.state.tally = Some(Tally {
13181            first_choice: BTreeMap::new(),
13182            borda: BTreeMap::new(),
13183            winner: winner.label,
13184            rankings: 0,
13185            unanimous_initial: false,
13186            deliberated: false,
13187            changed_votes: 0,
13188            unanimous_final: false,
13189            tie_break: None,
13190            judges: 0,
13191            present: 0,
13192            quorum: 0,
13193            met_quorum: true,
13194            uncontested: Some("fixture".into()),
13195        });
13196        runner.state.graft = Some(GraftRecord {
13197            pre: pre.clone(),
13198            post: None,
13199            lines: None,
13200            limit: None,
13201            agent: None,
13202            outcome: GraftOutcome::Running,
13203            requested: Vec::new(),
13204            adoption: None,
13205            adoption_error: None,
13206        });
13207        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13208        git::commit_all(&winner.worktree, "unfinished graft")
13209            .await
13210            .unwrap();
13211        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13212        let leftover = winner.worktree.join("leftover");
13213        std::fs::write(&leftover, "untrusted edits").unwrap();
13214        let path = runner.state.repo.join(".git/updater.lock");
13215        let external = std::fs::File::create(&path).unwrap();
13216        assert_eq!(
13217            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13218            0
13219        );
13220        assert!(
13221            runner
13222                .graft()
13223                .await
13224                .unwrap_err()
13225                .is::<crate::repo_lock::ResourceBlocked>()
13226        );
13227        assert!(matches!(
13228            runner.state.graft.as_ref().unwrap().outcome,
13229            GraftOutcome::Running
13230        ));
13231        assert_eq!(
13232            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13233            unfinished
13234        );
13235        assert!(leftover.exists());
13236        drop(external);
13237        std::fs::remove_file(&path).unwrap();
13238        std::fs::create_dir(&path).unwrap();
13239        assert!(runner.graft().await.is_err());
13240        assert!(matches!(
13241            runner.state.graft.as_ref().unwrap().outcome,
13242            GraftOutcome::Running
13243        ));
13244        assert_eq!(
13245            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13246            unfinished
13247        );
13248        assert!(leftover.exists());
13249        std::fs::remove_dir(&path).unwrap();
13250        runner.graft().await.unwrap();
13251        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13252        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13253        assert!(!leftover.exists());
13254    }
13255
13256    #[tokio::test]
13257    #[cfg(unix)]
13258    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13259        use std::os::fd::AsRawFd as _;
13260        let tmp = tempfile::tempdir().unwrap();
13261        let mut runner = registered_runner(tmp.path()).await;
13262        runner.state.prep_complete = Some(false);
13263        runner.state.config.graph.timeout_resource_wait = 1;
13264        runner.state.save().unwrap();
13265        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13266        assert_eq!(
13267            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13268            0
13269        );
13270        runner.execute().await.unwrap();
13271        let disk = RunState::load(&runner.state.id).unwrap();
13272        assert!(disk.driver_exited);
13273        assert_eq!(disk.driver_pid, None);
13274        assert_eq!(runner.state.status, RunStatus::Blocked);
13275        assert_eq!(
13276            runner.state.resource_block.as_ref().unwrap().kind,
13277            "repository coordination"
13278        );
13279        assert!(runner.state.prep_failure.is_none());
13280        assert!(runner.state.seats.is_empty());
13281        runner.pause.park();
13282        // Include headroom for Git subprocesses under parallel test load;
13283        // the repository contention deadline itself remains one second.
13284        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13285            .await
13286            .unwrap()
13287            .unwrap();
13288        assert!(runner.state.parked);
13289        assert!(runner.state.prep_failure.is_none());
13290        let disk = RunState::load(&runner.state.id).unwrap();
13291        assert!(disk.driver_exited);
13292        assert_eq!(disk.driver_pid, None);
13293    }
13294
13295    #[tokio::test]
13296    #[cfg(unix)]
13297    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13298        use std::os::fd::AsRawFd as _;
13299        let tmp = tempfile::tempdir().unwrap();
13300        let mut runner = registered_runner(tmp.path()).await;
13301        runner.state.prep_complete = Some(false);
13302        runner.state.config.graph.timeout_resource_wait = 30;
13303        runner.state.save().unwrap();
13304        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13305        assert_eq!(
13306            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13307            0
13308        );
13309        assert!(
13310            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13311                .await
13312                .is_err()
13313        );
13314        let disk = RunState::load(&runner.state.id).unwrap();
13315        assert!(disk.driver_exited);
13316        assert_eq!(disk.driver_pid, None);
13317        assert!(disk.status.resumable());
13318        assert!(runner.state.driver_exited);
13319        assert_eq!(runner.state.driver_pid, None);
13320        runner.state.save().unwrap();
13321        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13322    }
13323
13324    #[tokio::test]
13325    #[cfg(unix)]
13326    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13327        let tmp = tempfile::tempdir().unwrap();
13328        let mut runner = registered_runner(tmp.path()).await;
13329        let winner = runner.state.candidates[0].clone();
13330        let cache = tmp.path().join("cache");
13331        let started = tmp.path().join("pre-gate-started");
13332        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13333        runner.state.config.verify.gate =
13334            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13335        let home = crate::run::home();
13336        let owner =
13337            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13338        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13339            crate::cache::AcquireOutcome::Acquired(g) => g,
13340            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13341        };
13342        let mut gate = Box::pin(runner.run_gate(&winner));
13343        assert!(
13344            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13345                .await
13346                .is_err()
13347        );
13348        assert!(!started.exists(), "pre-gate must wait for the cache too");
13349        // Simulate the wave that already owns the cache entering invoke.
13350        // It must obtain the editor reservation and finish without waiting
13351        // for the gate that is queued behind its cache.
13352        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13353        let editor = crate::repo_lock::SeatLease::checkout_with(
13354            &winner.worktree,
13355            true,
13356            crate::config::test_coordination().as_ref(),
13357            &wait,
13358        )
13359        .await
13360        .unwrap();
13361        drop(editor);
13362        guard.release();
13363        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13364            .await
13365            .unwrap()
13366            .unwrap();
13367        drop(gate);
13368        assert!(outcomes.iter().all(CommandOutcome::ok));
13369        assert!(started.exists());
13370    }
13371
13372    #[tokio::test]
13373    #[cfg(unix)]
13374    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13375        use std::os::fd::AsRawFd as _;
13376
13377        ask_test_home();
13378        let tmp = tempfile::tempdir().unwrap();
13379        let repo = tmp.path().join("repo");
13380        std::fs::create_dir_all(&repo).unwrap();
13381        init_repo(&repo);
13382        let common = repo.join(".git");
13383        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13384        let mut runner = runner_at(RunStatus::Prep);
13385        runner.state.repo = repo.clone();
13386        runner.state.config.coordination = crate::config::test_coordination();
13387        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13388        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13389        runner.state.config.graph.advise = false;
13390        runner.roles.implementers = vec![spec("implement")];
13391        runner.roles.judges = vec![spec("judge")];
13392        runner.prep().await.unwrap();
13393        let winner = runner.state.candidates[0].clone();
13394        // Emulate edits already left by a completed seat. The fixture CLI
13395        // only reports completion; it performs no repository operations.
13396        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13397        let completed = tmp.path().join("seat-completed");
13398        runner.roles.implementers[0].command = vec![
13399            "sh".into(),
13400            "-c".into(),
13401            format!("touch '{}'; echo completed", completed.display()),
13402        ];
13403        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13404        assert_eq!(
13405            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13406            0
13407        );
13408        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13409        let mut implement = Box::pin(runner.implement());
13410        tokio::select! {
13411            result = &mut implement => panic!("rescue must wait: {result:?}"),
13412            result = tokio::time::timeout(Duration::from_secs(10), async {
13413                while !completed.exists() {
13414                    tokio::time::sleep(Duration::from_millis(20)).await;
13415                }
13416            }) => result.unwrap(),
13417        }
13418        assert!(
13419            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13420                .await
13421                .is_err()
13422        );
13423        assert_eq!(
13424            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13425            before
13426        );
13427        assert_eq!(
13428            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13429            0
13430        );
13431        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13432            .await
13433            .unwrap()
13434            .unwrap();
13435        drop(implement);
13436        assert_ne!(
13437            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13438            before
13439        );
13440        assert!(git::is_clean(&winner.worktree).await.unwrap());
13441
13442        let started = tmp.path().join("pre-gate-started");
13443        let proceed = tmp.path().join("pre-gate-proceed");
13444        let checked = tmp.path().join("gate-ran");
13445        runner.state.config.verify.pre_gate = vec![format!(
13446            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13447            started.display(),
13448            proceed.display()
13449        )];
13450        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13451        assert_eq!(
13452            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13453            0
13454        );
13455        let mut gate = Box::pin(runner.run_gate(&winner));
13456        assert!(
13457            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13458                .await
13459                .is_err()
13460        );
13461        assert!(!started.exists());
13462        assert!(!checked.exists());
13463        assert!(git::is_clean(&winner.worktree).await.unwrap());
13464        assert_eq!(
13465            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13466            0
13467        );
13468        tokio::select! {
13469            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13470            result = tokio::time::timeout(Duration::from_secs(10), async {
13471                while !started.exists() {
13472                    tokio::time::sleep(Duration::from_millis(20)).await;
13473                }
13474            }) => result.unwrap(),
13475        }
13476        assert_eq!(
13477            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13478            -1,
13479            "mechanical edits must hold the real lock"
13480        );
13481        std::fs::write(proceed, "ready").unwrap();
13482        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13483            .await
13484            .unwrap()
13485            .unwrap();
13486        drop(gate);
13487        assert!(outcomes.iter().all(CommandOutcome::ok));
13488        assert!(checked.exists());
13489        assert!(runner.state.pre_gate_commit.is_some());
13490        assert!(git::is_clean(&winner.worktree).await.unwrap());
13491        assert_eq!(
13492            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13493            0,
13494            "gate completion must release coordination"
13495        );
13496        drop(outsider);
13497        std::fs::remove_file(common.join("updater.lock")).unwrap();
13498        std::fs::create_dir(common.join("updater.lock")).unwrap();
13499        std::fs::remove_file(&started).unwrap();
13500        std::fs::remove_file(&checked).unwrap();
13501        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13502        let error = runner.run_gate(&winner).await.unwrap_err();
13503        assert!(
13504            error
13505                .to_string()
13506                .contains("acquire mandatory repository lock")
13507        );
13508        assert!(
13509            !started.exists(),
13510            "failed acquisition must prevent pre-gate"
13511        );
13512        assert!(!checked.exists(), "failed acquisition must prevent gate");
13513        assert!(git::is_clean(&winner.worktree).await.unwrap());
13514        assert_eq!(
13515            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13516            head
13517        );
13518    }
13519
13520    #[tokio::test]
13521    #[cfg(unix)]
13522    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13523        use std::os::fd::AsRawFd as _;
13524        use std::time::Duration;
13525
13526        ask_test_home();
13527        let tmp = tempfile::tempdir().unwrap();
13528        let repo = tmp.path().join("repo");
13529        std::fs::create_dir_all(&repo).unwrap();
13530        init_repo(&repo);
13531        let common = repo.join(".git");
13532        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13533        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13534        assert_eq!(
13535            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13536            0
13537        );
13538        let mut runner = runner_at(RunStatus::Prep);
13539        runner.state.repo = repo.clone();
13540        runner.state.config.coordination = crate::config::test_coordination();
13541        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13542        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13543        runner.state.config.graph.advise = true;
13544        runner.state.config.graph.advisors = 1;
13545        runner.roles.implementers = vec![spec("implement")];
13546        runner.roles.judges = vec![spec("judge")];
13547        let root = runner.state.worktree_root();
13548        let started = tmp.path().join("setup-started");
13549        let proceed = tmp.path().join("setup-proceed");
13550        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13551            run: Some(format!(
13552                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13553                started.display(),
13554                proceed.display()
13555            )),
13556            ..Default::default()
13557        }];
13558        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13559        let config = git::git(&repo, &["config", "--local", "--list"])
13560            .await
13561            .unwrap();
13562        let mut prep = Box::pin(runner.prep());
13563        assert!(
13564            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13565                .await
13566                .is_err()
13567        );
13568        assert!(
13569            !root.exists(),
13570            "contention must prevent even checkout creation"
13571        );
13572        assert!(!started.exists(), "contention must prevent setup");
13573        assert_eq!(
13574            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13575            branches
13576        );
13577        assert_eq!(
13578            git::git(&repo, &["config", "--local", "--list"])
13579                .await
13580                .unwrap(),
13581            config
13582        );
13583        drop(lock);
13584        tokio::select! {
13585            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13586            result = tokio::time::timeout(Duration::from_secs(10), async {
13587                while !started.exists() {
13588                    tokio::time::sleep(Duration::from_millis(20)).await;
13589                }
13590            }) => result.unwrap(),
13591        }
13592        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13593        assert_eq!(
13594            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13595            -1
13596        );
13597        assert!(
13598            tokio::time::timeout(
13599                Duration::from_millis(100),
13600                crate::repo_lock::SeatLease::for_checkout(
13601                    &repo,
13602                    true,
13603                    crate::config::test_coordination().as_ref()
13604                )
13605            )
13606            .await
13607            .is_err(),
13608            "preparation must also exclude local editors"
13609        );
13610        std::fs::write(proceed, "ready").unwrap();
13611        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13612            .await
13613            .unwrap()
13614            .unwrap();
13615        drop(prep);
13616        assert_eq!(runner.state.prep_complete, Some(true));
13617        assert!(root.join("judge-1").exists());
13618        assert!(root.join("advisor-1").exists());
13619        assert_eq!(
13620            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13621            0
13622        );
13623    }
13624
13625    #[tokio::test]
13626    #[cfg(unix)]
13627    async fn preparation_lock_failure_prevents_repository_changes() {
13628        ask_test_home();
13629        let tmp = tempfile::tempdir().unwrap();
13630        let repo = tmp.path().join("repo");
13631        std::fs::create_dir_all(&repo).unwrap();
13632        init_repo(&repo);
13633        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13634        // A directory at the required lock path makes opening the real lock
13635        // fail, rather than merely exercising an invalid registration.
13636        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13637        let mut runner = runner_at(RunStatus::Prep);
13638        runner.state.repo = repo.clone();
13639        runner.state.config.coordination = crate::config::test_coordination();
13640        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13641        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13642        runner.roles.implementers = vec![spec("implement")];
13643        let error = runner.prep().await.unwrap_err();
13644        assert!(
13645            error
13646                .to_string()
13647                .contains("acquire mandatory repository lock")
13648        );
13649        assert!(!runner.state.worktree_root().exists());
13650        assert!(runner.state.candidates.is_empty());
13651        assert!(!repo.join(".git/config.worktree").exists());
13652        assert_eq!(
13653            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13654            "* main"
13655        );
13656    }
13657
13658    #[tokio::test]
13659    async fn preparation_records_creation_intent_before_git_can_fail() {
13660        ask_test_home();
13661        let tmp = tempfile::tempdir().unwrap();
13662        let repo = tmp.path().join("repo");
13663        std::fs::create_dir_all(&repo).unwrap();
13664        init_repo(&repo);
13665        let mut runner = runner_at(RunStatus::Prep);
13666        runner.state.repo = repo.clone();
13667        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13668        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13669        runner.state.config.blind.commit_msg_hook = false;
13670        runner.roles.implementers = vec![spec("one")];
13671        let label = blind::assign_labels(1, runner.state.seed)[0];
13672        let branch = runner.state.branch_for(label);
13673        // A conflicting branch makes Git refuse creation before a candidate
13674        // can be registered. The durable intent must still identify it.
13675        git::git(&repo, &["branch", &branch]).await.unwrap();
13676        assert!(runner.prep().await.is_err());
13677        let saved = RunState::load(&runner.state.id).unwrap();
13678        assert!(saved.candidates.is_empty());
13679        let event = saved
13680            .events
13681            .iter()
13682            .find(|e| e.node == "prep-intent")
13683            .unwrap();
13684        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13685        assert_eq!(intent["branch"], branch);
13686        assert_eq!(intent["start"], runner.state.base_commit);
13687        assert_eq!(
13688            intent["worktree"],
13689            runner
13690                .state
13691                .worktree_root()
13692                .join(format!("cand-{label}"))
13693                .to_string_lossy()
13694                .as_ref()
13695        );
13696        assert_eq!(intent["picks"], serde_json::json!([]));
13697    }
13698
13699    #[tokio::test]
13700    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13701        ask_test_home();
13702        let tmp = tempfile::tempdir().unwrap();
13703        let repo = tmp.path().join("repo");
13704        std::fs::create_dir_all(&repo).unwrap();
13705        init_repo(&repo);
13706        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13707        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13708        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13709        git::git(&repo, &["add", "."]).await.unwrap();
13710        git::git(&repo, &["commit", "-m", "required seed"])
13711            .await
13712            .unwrap();
13713        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13714        git::git(&repo, &["checkout", "main"]).await.unwrap();
13715        let mut runner = runner_at(RunStatus::Prep);
13716        runner.state.repo = repo.clone();
13717        runner.state.base_commit = base.clone();
13718        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13719        runner.state.config.graph.advise = false;
13720        runner.state.config.blind.commit_msg_hook = false;
13721        runner.state.seeds = vec![refs::Seed {
13722            token: seed.clone(),
13723            sha: seed.clone(),
13724            kind: refs::SeedKind::Unmerged,
13725            branch: false,
13726            detail: String::new(),
13727        }];
13728        runner.roles.implementers = vec![spec("one"), spec("two")];
13729        runner.roles.judges = vec![spec("judge")];
13730        let label = blind::assign_labels(2, runner.state.seed)[0];
13731        let branch = runner.state.branch_for(label);
13732        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13733        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13734            .await
13735            .unwrap();
13736        runner.state.candidates.push(Candidate {
13737            index: 0,
13738            label,
13739            agent: "one".into(),
13740            branch,
13741            worktree,
13742            summary: String::new(),
13743            stat: String::new(),
13744            files: 0,
13745            commits: 0,
13746            empty: false,
13747            failed: None,
13748            verified_noop: None,
13749            duration_ms: 0,
13750            folded: false,
13751        });
13752        runner.state.save().unwrap();
13753        // Reload the checkpoint immediately before its first cherry-pick.
13754        runner.state = RunState::load(&runner.state.id).unwrap();
13755        assert_eq!(runner.state.prep_complete, Some(false));
13756        runner.prep().await.unwrap();
13757        assert_eq!(runner.state.prep_complete, Some(true));
13758        assert_eq!(runner.state.candidates.len(), 2);
13759        for candidate in &runner.state.candidates {
13760            assert_eq!(
13761                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13762                git::tree_of(&repo, &seed).await.unwrap()
13763            );
13764            assert_eq!(
13765                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13766                "required commit"
13767            );
13768        }
13769        assert!(runner.state.worktree_root().join("judge-1").exists());
13770        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13771        runner.prep().await.unwrap();
13772        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13773
13774        runner.state.prep_complete = Some(false);
13775        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13776            run: Some(format!(
13777                "test -f '{}'",
13778                tmp.path().join("dependency-ready").display()
13779            )),
13780            ..Default::default()
13781        }];
13782        assert!(runner.prep().await.is_err());
13783        assert!(runner.state.prep_failure.is_some());
13784        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13785        runner.state = RunState::load(&runner.state.id).unwrap();
13786        assert!(
13787            runner.prep().await.is_err(),
13788            "correcting the cause alone must not auto-retry"
13789        );
13790        let failed_at = runner.state.events.last().unwrap().at;
13791        assert!(
13792            runner.retry_preparation(failed_at).is_err(),
13793            "old approvals cannot unlock the hold"
13794        );
13795        let mut task = crate::queue::Task::new(
13796            "retry".into(),
13797            "retry setup".into(),
13798            repo.clone(),
13799            crate::queue::Source::Human,
13800        );
13801        task.start(runner.state.id.clone());
13802        task.hold_machine(Some("setup failed".into()));
13803        task.release_operator("cli", None);
13804        runner
13805            .retry_preparation(task.resumption.as_ref().unwrap().at)
13806            .unwrap();
13807        runner.prep().await.unwrap();
13808        assert!(runner.state.prep_failure.is_none());
13809        assert_eq!(runner.state.prep_complete, Some(true));
13810        assert_eq!(runner.state.candidates.len(), 2);
13811        assert!(runner.state.worktree_root().join("judge-1").exists());
13812        for candidate in &runner.state.candidates {
13813            assert_eq!(
13814                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13815                git::tree_of(&repo, &seed).await.unwrap()
13816            );
13817        }
13818    }
13819
13820    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13821        let mut heads = Vec::new();
13822        for candidate in candidates {
13823            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13824        }
13825        heads
13826    }
13827
13828    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13829        ask_test_home();
13830        let tmp = tempfile::tempdir().unwrap();
13831        let repo = tmp.path().join("repo");
13832        std::fs::create_dir(&repo).unwrap();
13833        init_repo(&repo);
13834        let mut runner = runner_at(RunStatus::Reviewing);
13835        runner.state.repo = repo.clone();
13836        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13837        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13838        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13839            let branch = format!("candidate-{label}");
13840            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13841            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13842                .await
13843                .unwrap();
13844            if label == 'B' {
13845                std::fs::write(worktree.join("change"), "winner").unwrap();
13846                git::rescue_commit(&worktree, "candidate change")
13847                    .await
13848                    .unwrap();
13849            }
13850            runner.state.candidates.push(Candidate {
13851                index,
13852                label,
13853                agent: "alpha".into(),
13854                branch,
13855                worktree,
13856                summary: "completed reply".into(),
13857                stat: String::new(),
13858                files: usize::from(label == 'B'),
13859                commits: usize::from(label == 'B'),
13860                empty: label == 'A',
13861                failed: None,
13862                verified_noop: None,
13863                duration_ms: 1,
13864                folded: false,
13865            });
13866        }
13867        runner.state.judge_skipped = true;
13868        runner.state.tally = Some(Tally {
13869            first_choice: BTreeMap::new(),
13870            borda: BTreeMap::new(),
13871            winner: 'B',
13872            rankings: 0,
13873            unanimous_initial: false,
13874            deliberated: false,
13875            changed_votes: 0,
13876            unanimous_final: false,
13877            tie_break: None,
13878            judges: 0,
13879            present: 0,
13880            quorum: 0,
13881            met_quorum: true,
13882            uncontested: Some("only B produced a change".into()),
13883        });
13884        runner.state.save().unwrap();
13885        (tmp, runner)
13886    }
13887
13888    #[tokio::test]
13889    #[cfg(unix)]
13890    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13891        for (solo, failed_checkpoint) in
13892            [(true, false), (false, false), (true, true), (false, true)]
13893        {
13894            let (_tmp, mut runner) = preservation_resume_fixture().await;
13895            if solo {
13896                runner.state.candidates.retain(|c| c.label == 'A');
13897            }
13898            runner.state.candidates[0].empty = false;
13899            runner.state.candidates[0].summary.clear();
13900            runner.state.tally = None;
13901            runner.state.judge_skipped = false;
13902            runner.state.status = RunStatus::Implementing;
13903            runner.state.config.disk.min_free_bytes = u64::MAX;
13904            let mut implementer = spec("alpha");
13905            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13906            runner.roles.implementers = vec![implementer];
13907            let patch = runner.state.dir().join("artifacts/cand-A.patch");
13908            if failed_checkpoint {
13909                std::fs::create_dir_all(&patch).unwrap();
13910            }
13911            runner.state.save().unwrap();
13912            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13913            runner.state = RunState::load(&runner.state.id).unwrap();
13914            assert_eq!(runner.state.status, RunStatus::Blocked);
13915            assert!(runner.state.interrupted_implementations.contains(&'A'));
13916            assert_eq!(runner.state.candidates[0].commits, 0);
13917            assert!(!runner.state.candidates[0].empty);
13918            assert!(runner.state.candidates[0].failed.is_none());
13919            if failed_checkpoint {
13920                assert!(runner.state.candidate_preservation.contains_key(&'A'));
13921                std::fs::remove_dir(patch).unwrap();
13922            }
13923            runner.state.resource_block = None;
13924            runner.state.config.disk.min_free_bytes = 0;
13925            runner.roles.implementers[0].command = vec![
13926                "sh".into(),
13927                "-c".into(),
13928                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13929            ];
13930            runner.implement().await.unwrap();
13931            assert!(runner.state.interrupted_implementations.is_empty());
13932            assert!(runner.state.candidate_preservation.is_empty());
13933            assert_eq!(runner.state.candidates[0].commits, 1);
13934            assert!(!runner.state.candidates[0].empty);
13935            assert!(
13936                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13937                    .unwrap()
13938                    .contains("retried empty interrupted seat")
13939            );
13940            if !solo {
13941                assert_eq!(runner.state.candidates[1].commits, 1);
13942            }
13943        }
13944    }
13945
13946    #[tokio::test]
13947    #[cfg(unix)]
13948    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13949        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13950            let (_tmp, mut runner) = preservation_resume_fixture().await;
13951            runner.state.candidates.retain(|c| c.label == 'A');
13952            runner.state.candidates[0].empty = false;
13953            runner.state.candidates[0].summary.clear();
13954            runner.state.tally = None;
13955            runner.state.judge_skipped = false;
13956            runner.state.status = RunStatus::Implementing;
13957            // Force the real post-spawn watchdog, independently of available
13958            // disk space. Dirty/committed work predates its first measurement.
13959            runner.state.config.disk.min_free_bytes = u64::MAX;
13960            let wt = runner.state.candidates[0].worktree.clone();
13961            std::fs::write(
13962                wt.join("interrupted-source"),
13963                "retain implementation work\n",
13964            )
13965            .unwrap();
13966            if committed {
13967                git::rescue_commit(&wt, "implementation checkpoint")
13968                    .await
13969                    .unwrap();
13970            }
13971            let lock = git::git(
13972                &wt,
13973                &[
13974                    "rev-parse",
13975                    "--path-format=absolute",
13976                    "--git-path",
13977                    "index.lock",
13978                ],
13979            )
13980            .await
13981            .unwrap();
13982            if block_rescue {
13983                std::fs::write(&lock, "fixture-owned lock").unwrap();
13984            }
13985            let mut implementer = spec("alpha");
13986            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13987            runner.roles.implementers = vec![implementer];
13988            runner.state.save().unwrap();
13989            let result = runner.implement().await;
13990            assert_eq!(result.is_err(), block_rescue);
13991            assert_eq!(
13992                runner.state.resource_block.as_ref().unwrap().kind,
13993                "disk pressure"
13994            );
13995            assert!(
13996                runner
13997                    .state
13998                    .events
13999                    .iter()
14000                    .any(|e| e.message.contains("agent execution interrupted"))
14001            );
14002            assert!(
14003                runner
14004                    .state
14005                    .dir()
14006                    .join("artifacts/candidate-A-recovery/staged.patch")
14007                    .exists()
14008            );
14009            if block_rescue {
14010                let saved = RunState::load(&runner.state.id).unwrap();
14011                assert!(saved.candidate_preservation.contains_key(&'A'));
14012                assert!(!saved.candidates[0].empty);
14013                assert_eq!(
14014                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14015                    "retain implementation work\n"
14016                );
14017                std::fs::remove_file(lock).unwrap();
14018                runner.state = saved;
14019            } else {
14020                assert_eq!(runner.state.status, RunStatus::Blocked);
14021                assert_eq!(runner.state.candidates[0].commits, 1);
14022            }
14023            // Recovery must save the candidate before deciding whether another
14024            // implementation is needed. No implementer is available on retry.
14025            runner.roles.implementers.clear();
14026            runner.state.config.disk.min_free_bytes = 0;
14027            runner.state.resource_block = None;
14028            runner.implement().await.unwrap();
14029            assert!(runner.state.candidate_preservation.is_empty());
14030            assert_eq!(runner.state.candidates[0].commits, 1);
14031            assert!(!runner.state.candidates[0].empty);
14032            assert!(git::is_clean(&wt).await.unwrap());
14033            assert!(
14034                crate::run::read_artifact(&runner.state, "cand-A.patch")
14035                    .unwrap()
14036                    .contains("retain implementation work")
14037            );
14038        }
14039    }
14040
14041    #[tokio::test]
14042    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14043        let (_tmp, mut runner) = preservation_resume_fixture().await;
14044        // Schema 15 already confirmed the unfolded no-change result.
14045        runner.implement().await.unwrap();
14046        assert!(runner.state.candidate_preservation.is_empty());
14047        assert!(
14048            !runner
14049                .state
14050                .dir()
14051                .join("artifacts/candidate-A-recovery")
14052                .exists()
14053        );
14054        runner.fold_losers().await.unwrap();
14055        assert!(!runner.state.candidates[0].worktree.exists());
14056        for schema in [14, crate::run::SCHEMA] {
14057            runner.state.schema = schema;
14058            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14059            runner.state.save().unwrap();
14060            runner.state = RunState::load(&runner.state.id).unwrap();
14061            runner.implement().await.unwrap();
14062            assert!(runner.state.candidate_preservation.is_empty());
14063            assert!(runner.state.candidates[0].folded);
14064            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14065        }
14066    }
14067
14068    #[tokio::test]
14069    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14070        let (_tmp, mut runner) = preservation_resume_fixture().await;
14071        let wt = runner.state.candidates[0].worktree.clone();
14072        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14073        runner.state.schema = 14;
14074        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14075        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14076        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14077        for _ in 0..2 {
14078            assert!(runner.implement().await.is_err());
14079            assert_eq!(runner.state.status, RunStatus::Blocked);
14080            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14081            assert!(
14082                runner.state.candidates[0].empty,
14083                "old verdict must remain unusable"
14084            );
14085            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14086            assert!(runner.state.judge_skipped);
14087            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14088            assert!(
14089                crate::run::read_artifact(&runner.state, "cand-A.patch")
14090                    .unwrap()
14091                    .contains("owned edits")
14092            );
14093            assert!(wt.exists());
14094            assert_eq!(
14095                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14096                    .await
14097                    .unwrap(),
14098                1
14099            );
14100            runner.state = RunState::load(&runner.state.id).unwrap();
14101        }
14102    }
14103
14104    #[tokio::test]
14105    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14106        let (_tmp, mut runner) = preservation_resume_fixture().await;
14107        let wt = runner.state.candidates[0].worktree.clone();
14108        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14109        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14110        let index_lock = PathBuf::from(
14111            git::git(
14112                &wt,
14113                &[
14114                    "rev-parse",
14115                    "--path-format=absolute",
14116                    "--git-path",
14117                    "index.lock",
14118                ],
14119            )
14120            .await
14121            .unwrap(),
14122        );
14123        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14124        runner.state.schema = 14;
14125        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14126        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14127        assert!(runner.implement().await.is_err());
14128        assert!(runner.after_implement().is_err());
14129        assert!(runner.fold_losers().await.is_err());
14130        assert_eq!(
14131            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14132            "failed seat edits"
14133        );
14134        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14135        runner.state = RunState::load(&runner.state.id).unwrap();
14136        std::fs::remove_file(index_lock).unwrap();
14137        // Recovery finds changes that the old uncontested result never judged.
14138        assert!(runner.implement().await.is_err());
14139        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14140        assert!(runner.state.candidates[0].empty);
14141        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14142        assert!(
14143            crate::run::read_artifact(&runner.state, "cand-A.patch")
14144                .unwrap()
14145                .contains("failed seat edits")
14146        );
14147        assert!(wt.exists());
14148    }
14149
14150    #[tokio::test]
14151    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14152        let (_tmp, mut runner) = preservation_resume_fixture().await;
14153        let repo = runner.state.repo.clone();
14154        let common = git::git(
14155            &repo,
14156            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14157        )
14158        .await
14159        .unwrap();
14160        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14161        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14162        assert!(runner.fold_losers().await.is_err());
14163        assert!(!runner.state.candidates[0].worktree.exists());
14164        assert!(!runner.state.candidates[0].folded);
14165        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14166        runner.state = RunState::load(&runner.state.id).unwrap();
14167        std::fs::remove_file(branch_lock).unwrap();
14168        let judge = runner.state.worktree_root().join("judge-1");
14169        git::worktree_add_detached(&repo, &judge, "main")
14170            .await
14171            .unwrap();
14172        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14173        runner.roles.judges = vec![spec("judge")];
14174        assert!(runner.fold_losers().await.is_err());
14175        runner.state = RunState::load(&runner.state.id).unwrap();
14176        assert!(
14177            runner.state.candidates[0].folded,
14178            "candidate progress saved before judge failure"
14179        );
14180        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14181        assert!(judge.join("uncommitted").exists());
14182        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14183        runner.fold_losers().await.unwrap();
14184        assert!(!judge.exists());
14185        runner.fold_losers().await.unwrap();
14186    }
14187
14188    #[tokio::test]
14189    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14190        let (_tmp, mut runner) = preservation_resume_fixture().await;
14191        runner.state.candidates.retain(|c| c.label == 'B');
14192        runner.state.tally = None;
14193        runner.state.judge_skipped = false;
14194        runner.state.prep_complete = Some(true);
14195        runner.state.config.graph.advise = false;
14196        runner.state.config.graph.selected_implementer = Some("alpha".into());
14197        let wt = runner.state.candidates[0].worktree.clone();
14198        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14199        runner
14200            .state
14201            .candidate_preservation
14202            .insert('B', "awaiting persistence".into());
14203        let index_lock = git::git(
14204            &wt,
14205            &[
14206                "rev-parse",
14207                "--path-format=absolute",
14208                "--git-path",
14209                "index.lock",
14210            ],
14211        )
14212        .await
14213        .unwrap();
14214        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14215        runner.state.save().unwrap();
14216        assert!(runner.execute_graph().await.is_err());
14217        assert_eq!(runner.state.status, RunStatus::Blocked);
14218        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14219        std::fs::remove_file(index_lock).unwrap();
14220        // Park at the next graph boundary: this tests the real selected-seat
14221        // hold guard without dispatching reviewers or publishing anything.
14222        for _ in 0..2 {
14223            runner.state = RunState::load(&runner.state.id).unwrap();
14224            runner.pause.park();
14225            runner.execute_graph().await.unwrap();
14226            assert_eq!(runner.state.status, RunStatus::Implementing);
14227            assert!(runner.state.candidate_preservation.is_empty());
14228            assert_eq!(runner.state.candidates[0].commits, 2);
14229            assert!(runner.state.tally.is_none());
14230        }
14231        runner.implement().await.unwrap();
14232        assert_eq!(runner.state.status, RunStatus::Judging);
14233        // A genuine selected-implementer hold must still require release.
14234        runner.state.status = RunStatus::Blocked;
14235        runner.state.candidates[0].failed = Some("implementation failed".into());
14236        runner.state.save().unwrap();
14237        let error = runner.execute_graph().await.unwrap_err();
14238        assert!(
14239            error
14240                .to_string()
14241                .contains("selected implementation is held")
14242        );
14243    }
14244
14245    #[tokio::test]
14246    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14247        let (_tmp, mut runner) = preservation_resume_fixture().await;
14248        let repo = runner.state.repo.clone();
14249        let candidate = runner.state.candidates[0].clone();
14250        // Remove only the disposable fixture directory, leaving Git's linked
14251        // worktree registration and branch intact, as in a vanished checkout.
14252        let registered = git::worktree_holding(&repo, &candidate.branch)
14253            .await
14254            .unwrap()
14255            .unwrap();
14256        assert_eq!(
14257            registered.canonicalize().unwrap(),
14258            candidate.worktree.canonicalize().unwrap()
14259        );
14260        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14261        assert_eq!(
14262            git::worktree_holding(&repo, &candidate.branch)
14263                .await
14264                .unwrap(),
14265            Some(registered)
14266        );
14267        assert!(
14268            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14269            "the stale registration initially keeps the branch checked out"
14270        );
14271        runner.fold_losers().await.unwrap();
14272        assert!(runner.state.candidates[0].folded);
14273        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14274        assert_eq!(
14275            git::worktree_holding(&repo, &candidate.branch)
14276                .await
14277                .unwrap(),
14278            None
14279        );
14280        runner.state = RunState::load(&runner.state.id).unwrap();
14281        runner.fold_losers().await.unwrap();
14282    }
14283
14284    #[tokio::test]
14285    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14286        let (_tmp, mut runner) = preservation_resume_fixture().await;
14287        let repo = runner.state.repo.clone();
14288        let candidate = runner.state.candidates[0].clone();
14289        let winner = runner.state.candidates[1].clone();
14290        let registered = git::worktree_holding(&repo, &candidate.branch)
14291            .await
14292            .unwrap()
14293            .unwrap();
14294        assert_eq!(
14295            registered.canonicalize().unwrap(),
14296            candidate.worktree.canonicalize().unwrap()
14297        );
14298        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14299        assert_eq!(
14300            git::worktree_holding(&repo, &candidate.branch)
14301                .await
14302                .unwrap(),
14303            Some(registered)
14304        );
14305        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14306
14307        for _ in 0..2 {
14308            fold_run(&mut runner.state, false, &crate::run::home())
14309                .await
14310                .unwrap();
14311            runner.state = RunState::load(&runner.state.id).unwrap();
14312            assert!(runner.state.candidates[0].folded);
14313            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14314            assert_eq!(
14315                git::worktree_holding(&repo, &candidate.branch)
14316                    .await
14317                    .unwrap(),
14318                None
14319            );
14320            assert!(!runner.state.candidates[1].folded);
14321            assert!(winner.worktree.exists());
14322            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14323        }
14324    }
14325
14326    #[tokio::test]
14327    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14328        ask_test_home();
14329        let tmp = tempfile::tempdir().unwrap();
14330        let repo = tmp.path().join("repo");
14331        std::fs::create_dir(&repo).unwrap();
14332        init_repo(&repo);
14333        let mut runner = runner_at(RunStatus::Implementing);
14334        runner.state.repo = repo.clone();
14335        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14336        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14337        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14338            let branch = format!("candidate-{label}");
14339            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14340            git::worktree_add_branch(&repo, &wt, &branch, "main")
14341                .await
14342                .unwrap();
14343            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14344            git::git(&wt, &["add", "README.md"]).await.unwrap();
14345            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14346            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14347            runner.state.candidates.push(Candidate {
14348                index,
14349                label,
14350                agent: "alpha".to_owned(),
14351                branch,
14352                worktree: wt,
14353                summary: "completed implementation".to_owned(),
14354                stat: String::new(),
14355                files: 0,
14356                commits: 0,
14357                empty: false,
14358                failed: None,
14359                verified_noop: None,
14360                duration_ms: 42,
14361                folded: false,
14362            });
14363            runner
14364                .state
14365                .candidate_preservation
14366                .insert(label, "awaiting persistence".to_owned());
14367        }
14368        let wt = runner.state.candidates[0].worktree.clone();
14369        let index_lock = git::git(
14370            &wt,
14371            &[
14372                "rev-parse",
14373                "--path-format=absolute",
14374                "--git-path",
14375                "index.lock",
14376            ],
14377        )
14378        .await
14379        .unwrap();
14380        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14381        runner.state.save().unwrap();
14382        assert!(runner.persist_candidates().await.is_err());
14383        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14384        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14385        assert_eq!(
14386            runner.state.candidates[1].commits, 1,
14387            "other results are still collected"
14388        );
14389        assert!(
14390            !runner.state.candidates[0].empty,
14391            "failed rescue is not no change"
14392        );
14393        assert_eq!(
14394            runner.state.candidates[0].summary,
14395            "completed implementation"
14396        );
14397        assert!(runner.after_implement().is_err());
14398        let home = tmp.path().join("fold-home");
14399        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14400        assert!(wt.join("new-file").exists());
14401        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14402        assert!(recovery.join("staged.patch").exists());
14403        assert!(recovery.join("unstaged.patch").exists());
14404        assert_eq!(
14405            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14406            "untracked edits\n"
14407        );
14408        runner.state.begin_pass();
14409        runner.state.end_pass_parked(ParkCause::Upgrade {
14410            from: "1.0".into(),
14411            to: Some("1.1".into()),
14412        });
14413        runner.state.save().unwrap();
14414        runner.state = RunState::load(&runner.state.id).unwrap();
14415        assert!(matches!(
14416            runner.state.passes.last().unwrap().end,
14417            Some(crate::run::PassEnd::Parked {
14418                cause: ParkCause::Upgrade { .. }
14419            })
14420        ));
14421        std::fs::remove_file(index_lock).unwrap();
14422        // No implementer is configured: a persistence retry cannot dispatch a seat.
14423        runner.implement().await.unwrap();
14424        assert!(runner.state.candidate_preservation.is_empty());
14425        assert!(
14426            runner
14427                .state
14428                .candidates
14429                .iter()
14430                .all(|c| c.commits == 1 && !c.empty)
14431        );
14432        assert!(git::is_clean(&wt).await.unwrap());
14433        assert!(
14434            crate::run::read_artifact(&runner.state, "cand-A.patch")
14435                .unwrap()
14436                .contains("later tracked edits")
14437        );
14438    }
14439
14440    #[tokio::test]
14441    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14442        ask_test_home();
14443        let tmp = tempfile::tempdir().unwrap();
14444        let repo = tmp.path().join("repo");
14445        std::fs::create_dir(&repo).unwrap();
14446        init_repo(&repo);
14447        let wt = tmp.path().join("candidate");
14448        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14449            .await
14450            .unwrap();
14451        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14452        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14453        assert!(wt.join("uncommitted").exists());
14454        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14455        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14456        let mut runner = runner_at(RunStatus::Implementing);
14457        runner.state.repo = repo.clone();
14458        runner.state.base_commit = "missing-base".to_owned();
14459        runner.state.candidates.push(Candidate {
14460            index: 0,
14461            label: 'A',
14462            agent: "alpha".into(),
14463            branch: "candidate".into(),
14464            worktree: wt.clone(),
14465            summary: "completed".into(),
14466            stat: String::new(),
14467            files: 0,
14468            commits: 0,
14469            empty: false,
14470            failed: None,
14471            verified_noop: None,
14472            duration_ms: 1,
14473            folded: false,
14474        });
14475        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14476        assert!(
14477            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14478                .await
14479                .is_err()
14480        );
14481        assert!(!runner.state.candidates[0].folded);
14482        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14483        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14484        runner
14485            .state
14486            .candidate_preservation
14487            .insert('A', "pending".into());
14488        assert!(runner.persist_candidates().await.is_err());
14489        assert!(!runner.state.candidates[0].empty);
14490        assert!(!runner.state.candidates[0].folded);
14491        assert!(wt.exists());
14492        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14493        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14494        std::fs::create_dir_all(&patch_path).unwrap();
14495        assert!(
14496            runner.persist_candidates().await.is_err(),
14497            "artifact failure stays pending"
14498        );
14499        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14500        assert!(!runner.state.candidates[0].empty);
14501        std::fs::remove_dir(patch_path).unwrap();
14502        runner.persist_candidates().await.unwrap();
14503        assert!(
14504            runner.state.candidates[0].empty,
14505            "only successful inspection confirms no change"
14506        );
14507    }
14508
14509    fn init_repo(dir: &Path) {
14510        let run = |args: &[&str]| {
14511            let out = std::process::Command::new("git")
14512                .args(args)
14513                .current_dir(dir)
14514                .quiet()
14515                .output()
14516                .expect("spawn git");
14517            assert!(
14518                out.status.success(),
14519                "git {args:?} failed: {}",
14520                String::from_utf8_lossy(&out.stderr)
14521            );
14522        };
14523        run(&["init", "-b", "main"]);
14524        run(&["config", "user.name", "magi test"]);
14525        run(&["config", "user.email", "magi@example.com"]);
14526        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14527        run(&["add", "-A"]);
14528        run(&["commit", "-m", "init"]);
14529    }
14530
14531    // `settle_questions` is what closes the ghost the phone showed: a run's
14532    // seat asked something, the run then ended, and nothing was left to
14533    // abandon the question it left `open`. `HOME` is a process-wide
14534    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14535    // first time it runs in the binary — every test below still reaches the
14536    // same directory whichever call won, and each gets its own run id from
14537    // `RunState::new`, so they never collide there.
14538    fn ask_test_home() {
14539        crate::run::pin_test_home();
14540    }
14541
14542    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14543    /// reads nothing else off it.
14544    fn runner_at(status: RunStatus) -> Runner {
14545        let mut state = RunState::new(
14546            PathBuf::from("/nonexistent/repo"),
14547            "main".to_owned(),
14548            "deadbeef".to_owned(),
14549            "task".to_owned(),
14550            Config::default(),
14551        );
14552        // These fixtures share one home and save concurrently. Short clock
14553        // IDs can collide, making an ownership test read another test's run.
14554        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14555        state.id = format!(
14556            "20000101-000000-{:04x}",
14557            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14558        );
14559        state.status = status;
14560        Runner {
14561            state,
14562            roles: ResolvedRoles {
14563                implementers: Vec::new(),
14564                judges: Vec::new(),
14565                reviewers: Vec::new(),
14566                fixer: None,
14567                conductor: conductor(),
14568                implementer_roster: Vec::new(),
14569                judge_roster: Vec::new(),
14570                reviewer_roster: Vec::new(),
14571            },
14572            sem: Arc::new(Semaphore::new(1)),
14573            pause: Pause::new(),
14574            interrupt: Pause::new(),
14575        }
14576    }
14577
14578    #[test]
14579    fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14580        crate::run::pin_test_home();
14581        let mut runner = runner_at(RunStatus::Gating);
14582        runner.state.save().unwrap();
14583        // run.json can commit, but the subsequent lease metadata write fails.
14584        // Cleanup must run after releasing the short ownership lock.
14585        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14586        assert!(runner.claim_driver().is_err());
14587        assert!(runner.state.driver_exited);
14588        assert_eq!(runner.state.driver_pid, None);
14589        let disk = RunState::load(&runner.state.id).unwrap();
14590        assert!(disk.driver_exited);
14591        assert_eq!(disk.driver_pid, None);
14592        assert_eq!(disk.status, RunStatus::Gating);
14593    }
14594
14595    #[tokio::test]
14596    async fn an_early_graph_error_releases_its_driver() {
14597        crate::run::pin_test_home();
14598        let mut runner = runner_at(RunStatus::Blocked);
14599        runner.state.config.graph.selected_implementer = Some("selected".into());
14600        runner.state.save().unwrap();
14601        assert!(runner.execute().await.is_err());
14602        let disk = RunState::load(&runner.state.id).unwrap();
14603        assert!(disk.driver_exited);
14604        assert_eq!(disk.driver_pid, None);
14605        assert_eq!(disk.status, RunStatus::Blocked);
14606    }
14607
14608    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14609        crate::run::MergeRepair {
14610            question: "repair-question".into(),
14611            head: "old-head".into(),
14612            pr: "https://github.com/o/r/pull/42".into(),
14613            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14614            gate: vec![outcome(Some(0), false)],
14615            gate_head: Some("old-head".into()),
14616            gate_fixes: Vec::new(),
14617            pre_gate: Vec::new(),
14618            phase,
14619            failure: None,
14620        }
14621    }
14622
14623    #[tokio::test]
14624    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14625        crate::run::pin_test_home();
14626        let mut runner = runner_at(RunStatus::Landing);
14627        runner.state.merge_repairs.push(merge_repair_request(
14628            crate::run::MergeRepairPhase::Requested,
14629        ));
14630        runner.state.save().unwrap();
14631        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14632        assert!(
14633            runner
14634                .execute()
14635                .await
14636                .unwrap_err()
14637                .to_string()
14638                .contains("already running")
14639        );
14640        assert_eq!(
14641            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14642            crate::run::MergeRepairPhase::Requested
14643        );
14644        drop(claim);
14645        let mut disk = runner.state.clone();
14646        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14647        disk.save().unwrap();
14648        assert!(
14649            runner
14650                .execute()
14651                .await
14652                .unwrap_err()
14653                .to_string()
14654                .contains("decision changed")
14655        );
14656        assert_eq!(
14657            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14658            crate::run::MergeRepairPhase::Complete
14659        );
14660    }
14661
14662    #[test]
14663    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14664        crate::run::pin_test_home();
14665        let mut runner = runner_at(RunStatus::Landing);
14666        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14667        runner.state.reviews = request.reviews.clone();
14668        runner.state.gate = request.gate.clone();
14669        runner.state.gate_ran = true;
14670        runner.state.land_approval = Some(crate::run::LandApproval {
14671            question: "old".into(),
14672            head: "old-head".into(),
14673        });
14674        runner.state.merge_repairs.push(request);
14675        runner.begin_merge_repair_review(0).unwrap();
14676        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14677        assert!(!runner.state.gate_status().ok());
14678        assert!(runner.state.land_approval.is_none());
14679        let disk = RunState::load(&runner.state.id).unwrap();
14680        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14681        assert!(disk.merge_repairs[0].gate[0].ok());
14682        assert_eq!(
14683            disk.merge_repairs[0].phase,
14684            crate::run::MergeRepairPhase::Reviewing
14685        );
14686        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14687        assert!(disk.reviews.is_empty());
14688    }
14689
14690    #[tokio::test]
14691    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14692        crate::run::pin_test_home();
14693        for phase in [
14694            crate::run::MergeRepairPhase::Fixing,
14695            crate::run::MergeRepairPhase::Requested,
14696        ] {
14697            let mut runner = runner_at(RunStatus::Landing);
14698            runner.state.config.graph.land_rounds = 0;
14699            runner.state.merge_repairs.push(merge_repair_request(phase));
14700            runner.state.save().unwrap();
14701            assert!(!runner.repair_merge_review().await.unwrap());
14702            assert_eq!(runner.state.status, RunStatus::Blocked);
14703            assert_eq!(
14704                runner.state.merge_repairs[0].phase,
14705                crate::run::MergeRepairPhase::Failed
14706            );
14707            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14708            assert!(
14709                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14710                    "interrupted"
14711                } else {
14712                    "budget exhausted"
14713                })
14714            );
14715            assert!(runner.state.merge.is_none());
14716        }
14717    }
14718
14719    /// `park_here` folding in the reason `Pause::park_because` recorded -
14720    /// this is what lets an operator reading a run's events tell an
14721    /// interrupt-driven park from an ordinary shutdown park.
14722    #[test]
14723    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14724        crate::run::pin_test_home();
14725        let mut runner = runner_at(RunStatus::Implementing);
14726        let interrupt = Pause::new();
14727        runner.watch_interrupt(interrupt.clone());
14728
14729        interrupt.park_because("task a1b2 asked to run first");
14730
14731        assert!(runner.park_here().expect("park_here"));
14732        assert!(runner.state.parked);
14733        let last = runner.state.events.last().expect("a park event");
14734        assert_eq!(last.node, "park");
14735        assert!(
14736            last.message.contains("task a1b2 asked to run first"),
14737            "expected the interrupt reason in {:?}",
14738            last.message
14739        );
14740    }
14741
14742    /// The cause is stored on the pass when it parks, and the first cause a
14743    /// shared pause was given is the one that stays.
14744    #[test]
14745    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14746        crate::run::pin_test_home();
14747        let mut runner = runner_at(RunStatus::Implementing);
14748        runner.state.begin_pass();
14749        let shutdown = Pause::new();
14750        runner.on_pause(shutdown.clone());
14751        let upgrade = ParkCause::Upgrade {
14752            from: "0.121.0".to_owned(),
14753            to: Some("0.121.1".to_owned()),
14754        };
14755        shutdown.park_for(upgrade.clone());
14756        shutdown.park_for(ParkCause::Operator);
14757
14758        assert!(runner.park_here().expect("park_here"));
14759        assert_eq!(
14760            runner.state.passes[0].end,
14761            Some(crate::run::PassEnd::Parked { cause: upgrade })
14762        );
14763        let last = runner.state.events.last().expect("a park event");
14764        assert!(last.message.contains("0.121.0"), "{}", last.message);
14765    }
14766
14767    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14768    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14769    /// cleared) must not make a *different* run - one only watching its own,
14770    /// unshared interrupt `Pause` - see itself as parked. If a future change
14771    /// ever collapsed these back into one handle, the interrupt scheduler
14772    /// would park every run for the rest of the daemon's life, not just the
14773    /// one it meant to interrupt.
14774    #[test]
14775    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14776        crate::run::pin_test_home();
14777        let mut runner = runner_at(RunStatus::Implementing);
14778        let shutdown = Pause::new();
14779        runner.on_pause(shutdown.clone());
14780        let interrupt = Pause::new();
14781        runner.watch_interrupt(interrupt.clone());
14782
14783        // Nobody has asked for anything yet.
14784        assert!(!runner.park_here().expect("park_here"));
14785        assert!(!runner.state.parked);
14786
14787        // Only the interrupt handle fires; the shutdown handle stays clear.
14788        interrupt.park_because("test");
14789        assert!(!shutdown.parked());
14790        assert!(runner.park_here().expect("park_here"));
14791    }
14792
14793    /// The property every prior attempt at this feature failed to pin down:
14794    /// asking a run to park while one of its nodes has a real, in-flight
14795    /// async operation running (an agent call, in production) must not cut
14796    /// that operation short. `park_here` is only ever consulted *between*
14797    /// `execute`'s node calls - see its own doc - so nothing inside a node
14798    /// can observe a park request until the node itself returns. This proves
14799    /// that structurally, with real `tokio` concurrency and a channel
14800    /// handshake (never a sleep, which would only prove "usually", not
14801    /// "cannot"): the "node" below reports that it has genuinely started,
14802    /// and only then is the park requested; the node still has to be told to
14803    /// finish before `park_here` is ever called, exactly mirroring every
14804    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14805    /// in `execute`.
14806    #[tokio::test]
14807    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14808        crate::run::pin_test_home();
14809        let mut runner = runner_at(RunStatus::Implementing);
14810        let interrupt = Pause::new();
14811        runner.watch_interrupt(interrupt.clone());
14812
14813        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14814        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14815
14816        // Stands in for one node's in-flight agent call: it proves it has
14817        // genuinely started, then blocks - exactly as a spawned CLI process
14818        // does - until told to finish.
14819        let node = async move {
14820            started_tx.send(()).expect("send started");
14821            finish_rx.await.expect("recv finish");
14822            "node finished"
14823        };
14824
14825        let interrupter = async move {
14826            started_rx.await.expect("recv started");
14827            // The call is now genuinely in flight. Ask it to park.
14828            interrupt.park_because("higher-priority task waiting");
14829            // Nothing the node does can observe this yet - there is no
14830            // check inside it, by construction - so let the executor run
14831            // anything pending and then let the node finish on its own.
14832            tokio::task::yield_now().await;
14833            finish_tx.send(()).expect("send finish");
14834        };
14835
14836        let (node_result, ()) = tokio::join!(node, interrupter);
14837        assert_eq!(
14838            node_result, "node finished",
14839            "the in-flight call ran to completion"
14840        );
14841
14842        // Only now, at the boundary the real `execute` would check right
14843        // after this node, does the park take effect.
14844        assert!(runner.park_here().expect("park_here"));
14845        assert!(runner.state.parked);
14846    }
14847
14848    /// A run parked mid-competition carries every field it had accumulated
14849    /// through the exact same disk round-trip an ordinary resume uses -
14850    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
14851    /// Nothing about parking for an interrupt is a special case of that path;
14852    /// this is what proves it rather than assuming it.
14853    #[test]
14854    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14855        crate::run::pin_test_home();
14856        let mut runner = runner_at(RunStatus::Judging);
14857        // `Runner::resume` re-resolves roles from the saved config, which
14858        // refuses an empty roster - give it the same minimal one `conductor`
14859        // itself uses.
14860        runner.state.config.agents = vec![conductor()];
14861        runner.state.candidates = vec![Candidate {
14862            index: 0,
14863            label: 'A',
14864            agent: "alpha".to_owned(),
14865            branch: "magi/x/A".to_owned(),
14866            worktree: PathBuf::from("/nonexistent/worktree"),
14867            summary: "did the thing".to_owned(),
14868            stat: "1 file changed".to_owned(),
14869            files: 1,
14870            commits: 1,
14871            empty: false,
14872            failed: None,
14873            verified_noop: None,
14874            duration_ms: 1234,
14875            folded: false,
14876        }];
14877        let run_id = runner.state.id.clone();
14878
14879        let interrupt = Pause::new();
14880        runner.watch_interrupt(interrupt.clone());
14881        interrupt.park_because("task c3d4 asked to run first");
14882        assert!(runner.park_here().expect("park_here"));
14883
14884        let resumed = Runner::resume(&run_id).expect("resume");
14885        assert_eq!(resumed.state.candidates.len(), 1);
14886        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14887        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14888        assert_eq!(resumed.state.status, runner.state.status);
14889        assert!(
14890            resumed.state.parked,
14891            "still parked until `execute` actually walks the graph again"
14892        );
14893        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14894    }
14895
14896    /// A fresh open question on `run`, stored and handed back for assertions.
14897    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14898        let mut q = ask::Question::new(
14899            run.to_owned(),
14900            "implement".to_owned(),
14901            "impl-A".to_owned(),
14902            "Which storage backend should the cache use?".to_owned(),
14903            String::new(),
14904            vec!["SQLite".to_owned(), "Redis".to_owned()],
14905        );
14906        store.put(&mut q).unwrap();
14907        q
14908    }
14909
14910    #[test]
14911    fn a_failed_runs_open_question_is_abandoned() {
14912        ask_test_home();
14913        let store = ask::Questions::open();
14914        let mut runner = runner_at(RunStatus::Failed);
14915        let run = runner.state.id.clone();
14916        let q = ask_open_question(&store, &run);
14917
14918        runner.settle_questions();
14919
14920        let back = store.get(&q.id).unwrap();
14921        assert!(
14922            !back.status.open(),
14923            "the seat that asked died with the run; nobody is left to read an answer"
14924        );
14925        assert!(
14926            back.detail.contains(&run) && back.detail.contains("failed"),
14927            "the reason names what the run became, not just that it is gone: {}",
14928            back.detail
14929        );
14930    }
14931
14932    #[test]
14933    fn a_merged_runs_open_question_is_abandoned_too() {
14934        ask_test_home();
14935        let store = ask::Questions::open();
14936        // A run that finishes cleanly still leaves nobody to read an answer -
14937        // this is not only a failure-path cleanup.
14938        for status in [RunStatus::Merged, RunStatus::Ready] {
14939            let mut runner = runner_at(status);
14940            let run = runner.state.id.clone();
14941            let q = ask_open_question(&store, &run);
14942
14943            runner.settle_questions();
14944
14945            let back = store.get(&q.id).unwrap();
14946            assert!(
14947                !back.status.open(),
14948                "{status:?} run's question must not outlive the run"
14949            );
14950        }
14951    }
14952
14953    #[test]
14954    fn a_still_resumable_runs_open_question_is_left_alone() {
14955        ask_test_home();
14956        let store = ask::Questions::open();
14957        // `Blocked` and `Stalled` can still be resumed — the candidates, the
14958        // review round and the seat sessions are all still on disk — so a
14959        // question asked mid-round may yet get a real answer from a real
14960        // resume. Sweeping it here would be exactly the failure mode this
14961        // whole feature exists to avoid on the other side.
14962        for status in [RunStatus::Blocked, RunStatus::Stalled] {
14963            let mut runner = runner_at(status);
14964            let run = runner.state.id.clone();
14965            let q = ask_open_question(&store, &run);
14966
14967            runner.settle_questions();
14968
14969            let back = store.get(&q.id).unwrap();
14970            assert!(
14971                back.status.open(),
14972                "{status:?} is still alive; the question must still be waiting"
14973            );
14974        }
14975    }
14976
14977    #[test]
14978    fn settle_questions_never_touches_an_already_answered_question() {
14979        ask_test_home();
14980        let store = ask::Questions::open();
14981        let mut runner = runner_at(RunStatus::Failed);
14982        let run = runner.state.id.clone();
14983        let mut q = ask_open_question(&store, &run);
14984        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14985            .unwrap();
14986        store.put(&mut q).unwrap();
14987
14988        // Called twice, the way a crash-recovered daemon reclaim and the
14989        // graph's own cleanup both can for the same run — `abandon_for_run`
14990        // only ever touches what is still open, so this must be inert both
14991        // times, not merely the second.
14992        runner.settle_questions();
14993        runner.settle_questions();
14994
14995        let back = store.get(&q.id).unwrap();
14996        assert_eq!(
14997            back.status,
14998            ask::QuestionStatus::Answered,
14999            "a real answer is a decision on record, never overwritten by a sweep"
15000        );
15001    }
15002
15003    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15004    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15005    /// without merging, whose winner is still the operator's answer to read.
15006    /// Nothing previously called `fold_run` itself with a real `tally`, so
15007    /// this is the first test to pin down the one distinction the whole
15008    /// automatic-fold feature depends on: the winner's worktree and branch
15009    /// must survive, everything else sharing the run's worktree bay - a
15010    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15011    /// second sweep treats every non-winner directory under the bay alike -
15012    /// must not.
15013    #[tokio::test]
15014    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15015        crate::run::pin_test_home();
15016        let tmp = tempfile::tempdir().expect("tempdir");
15017        let repo = tmp.path().join("repo");
15018        std::fs::create_dir_all(&repo).unwrap();
15019        init_repo(&repo);
15020
15021        let mut config = Config::default();
15022        config.graph.worktree_root = Some(tmp.path().join("wt"));
15023
15024        let mut state = RunState::new(
15025            repo.clone(),
15026            "main".to_owned(),
15027            "deadbeef".to_owned(),
15028            "task".to_owned(),
15029            config,
15030        );
15031        let root = state.worktree_root();
15032        let wt_a = root.join("cand-A");
15033        let wt_b = root.join("cand-B");
15034        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15035            .await
15036            .expect("worktree A");
15037        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15038            .await
15039            .expect("worktree B");
15040
15041        state.candidates = vec![
15042            Candidate {
15043                index: 0,
15044                label: 'A',
15045                agent: "alpha".to_owned(),
15046                branch: "magi/x/A".to_owned(),
15047                worktree: wt_a.clone(),
15048                summary: String::new(),
15049                stat: String::new(),
15050                files: 0,
15051                commits: 0,
15052                empty: false,
15053                failed: None,
15054                verified_noop: None,
15055                duration_ms: 0,
15056                folded: false,
15057            },
15058            Candidate {
15059                index: 1,
15060                label: 'B',
15061                agent: "beta".to_owned(),
15062                branch: "magi/x/B".to_owned(),
15063                worktree: wt_b.clone(),
15064                summary: String::new(),
15065                stat: String::new(),
15066                files: 0,
15067                commits: 0,
15068                empty: false,
15069                failed: None,
15070                verified_noop: None,
15071                duration_ms: 0,
15072                folded: false,
15073            },
15074        ];
15075        state.tally = Some(Tally {
15076            first_choice: BTreeMap::from([('A', 1)]),
15077            borda: BTreeMap::new(),
15078            winner: 'A',
15079            rankings: 1,
15080            unanimous_initial: true,
15081            deliberated: false,
15082            changed_votes: 0,
15083            unanimous_final: true,
15084            tie_break: None,
15085            judges: 1,
15086            present: 1,
15087            quorum: 1,
15088            met_quorum: true,
15089            uncontested: None,
15090        });
15091        state.status = RunStatus::Ready;
15092
15093        fold_run(&mut state, false, &crate::run::home())
15094            .await
15095            .expect("fold_run");
15096
15097        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15098        assert!(
15099            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15100            "the unmerged winner's branch survives"
15101        );
15102        assert!(
15103            !state.candidates[0].folded,
15104            "the winner is not marked folded"
15105        );
15106
15107        assert!(!wt_b.exists(), "the loser's worktree is removed");
15108        assert!(
15109            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15110            "the loser's branch is removed"
15111        );
15112        assert!(state.candidates[1].folded, "the loser is marked folded");
15113    }
15114
15115    #[tokio::test]
15116    #[cfg(unix)]
15117    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15118        use std::os::fd::AsRawFd as _;
15119        use std::time::Duration;
15120
15121        let tmp = tempfile::tempdir().unwrap();
15122        let repo = tmp.path().join("repo");
15123        std::fs::create_dir_all(&repo).unwrap();
15124        init_repo(&repo);
15125        let common = repo.join(".git");
15126        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15127        let home = tmp.path().join("home");
15128        let mut config = Config::default();
15129        config.coordination = crate::config::test_coordination();
15130        config.graph.worktree_root = Some(tmp.path().join("wt"));
15131        config.graph.timeout_resource_wait = 1;
15132        let mut state = RunState::new(
15133            repo.clone(),
15134            "main".into(),
15135            "deadbeef".into(),
15136            "task".into(),
15137            config,
15138        );
15139        let root = state.worktree_root();
15140        let candidate = root.join("cand-A");
15141        let reviewer = root.join("review-1");
15142        let branch = state.branch_for('A');
15143        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15144            .await
15145            .unwrap();
15146        git::worktree_add_detached(&repo, &reviewer, "main")
15147            .await
15148            .unwrap();
15149        git::acquire_worktree_config(&repo).await.unwrap();
15150        state.enabled_worktree_config = true;
15151        state.status = RunStatus::Ready;
15152        state.candidates.push(Candidate {
15153            index: 0,
15154            label: 'A',
15155            agent: "implement".into(),
15156            branch: branch.clone(),
15157            worktree: candidate.clone(),
15158            summary: String::new(),
15159            stat: String::new(),
15160            files: 0,
15161            commits: 0,
15162            empty: false,
15163            failed: None,
15164            verified_noop: None,
15165            duration_ms: 0,
15166            folded: false,
15167        });
15168        state.save_under(&home).unwrap();
15169        let config_before = std::fs::read(common.join("config")).unwrap();
15170        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15171            .await
15172            .unwrap();
15173        // A separate OS handle bypasses the launcher's local gate, exactly as
15174        // an external updater does, and must still prevent all folding work.
15175        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15176        assert_eq!(
15177            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15178            0
15179        );
15180        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15181        assert!(
15182            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15183                .await
15184                .is_err()
15185        );
15186        assert!(candidate.exists());
15187        assert!(reviewer.exists());
15188        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15189        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15190        assert_eq!(
15191            git::git(&repo, &["worktree", "list", "--porcelain"])
15192                .await
15193                .unwrap(),
15194            worktrees_before
15195        );
15196        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15197            .await
15198            .unwrap()
15199            .unwrap_err();
15200        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15201        drop(fold);
15202        assert!(candidate.exists() && reviewer.exists());
15203        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15204        assert_eq!(
15205            state.resource_block.as_ref().unwrap().kind,
15206            "repository coordination"
15207        );
15208        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15209        drop(external);
15210        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15211        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15212            .await
15213            .unwrap()
15214            .unwrap();
15215        drop(fold);
15216        assert!(removed.contains(&branch));
15217        assert!(!root.exists());
15218        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15219        assert!(state.candidates[0].folded);
15220        assert!(!state.enabled_worktree_config);
15221        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15222        let enabled = git::git_raw(
15223            &repo,
15224            &["config", "--local", "--get", "extensions.worktreeConfig"],
15225        )
15226        .await
15227        .unwrap();
15228        assert!(!enabled.ok(), "the last config reference was released");
15229    }
15230
15231    /// A branch handed to a later run is that run's (and its pull request's):
15232    /// folding the run that released it must not delete it.
15233    #[tokio::test]
15234    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15235        let tmp = tempfile::tempdir().expect("tempdir");
15236        let repo = tmp.path().join("repo");
15237        std::fs::create_dir_all(&repo).unwrap();
15238        init_repo(&repo);
15239        let home = tmp.path().join("home");
15240
15241        let mut config = Config::default();
15242        config.graph.worktree_root = Some(tmp.path().join("wt"));
15243        let mut state = RunState::new(
15244            repo.clone(),
15245            "main".to_owned(),
15246            "deadbeef".to_owned(),
15247            "task".to_owned(),
15248            config,
15249        );
15250        // The worktree is already gone (released); the branch survives.
15251        git::git(&repo, &["branch", "magi/x/A", "main"])
15252            .await
15253            .expect("branch");
15254        state.candidates = vec![Candidate {
15255            index: 0,
15256            label: 'A',
15257            agent: "alpha".to_owned(),
15258            branch: "magi/x/A".to_owned(),
15259            worktree: state.worktree_root().join("cand-A"),
15260            summary: String::new(),
15261            stat: String::new(),
15262            files: 0,
15263            commits: 0,
15264            empty: false,
15265            failed: None,
15266            verified_noop: None,
15267            duration_ms: 0,
15268            folded: true,
15269        }];
15270        state.released_to = Some("20260901-000000-new1".to_owned());
15271        state.released_branches = vec!["magi/x/A".to_owned()];
15272
15273        fold_run(&mut state, true, &home).await.expect("fold_run");
15274
15275        assert!(
15276            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15277            "the handed-over branch survives a fold"
15278        );
15279    }
15280
15281    /// A winner with nothing ahead of the base is caught before `gh` is ever
15282    /// asked for a pull request, and the message carries what the task's
15283    /// references resolved to.
15284    #[tokio::test]
15285    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15286        let tmp = tempfile::tempdir().expect("tempdir");
15287        let repo = tmp.path().join("repo");
15288        std::fs::create_dir_all(&repo).unwrap();
15289        init_repo(&repo);
15290        let run = |args: &[&str]| {
15291            let out = std::process::Command::new("git")
15292                .quiet()
15293                .args(args)
15294                .current_dir(&repo)
15295                .output()
15296                .expect("spawn git");
15297            assert!(out.status.success(), "git {args:?}");
15298        };
15299        run(&["branch", "magi/x/A"]);
15300        run(&["checkout", "-q", "-b", "magi/x/B"]);
15301        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15302        run(&["add", "-A"]);
15303        run(&["commit", "-q", "-m", "work"]);
15304        run(&["checkout", "-q", "main"]);
15305
15306        // A pull request is compared against the remote's base, so the
15307        // fixture needs one. Before it exists nothing can be read, and the
15308        // answer must be "not empty".
15309        let probe = RunState::new(
15310            repo.clone(),
15311            "main".to_owned(),
15312            "deadbeef".to_owned(),
15313            "task".to_owned(),
15314            Config::default(),
15315        );
15316        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15317        let bare = tmp.path().join("origin.git");
15318        let out = std::process::Command::new("git")
15319            .quiet()
15320            .args(["init", "-q", "--bare"])
15321            .arg(&bare)
15322            .output()
15323            .expect("spawn git");
15324        assert!(out.status.success(), "git init --bare");
15325        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15326        run(&["push", "-q", "origin", "main"]);
15327
15328        let mut state = RunState::new(
15329            repo.clone(),
15330            "main".to_owned(),
15331            "deadbeef".to_owned(),
15332            "task".to_owned(),
15333            Config::default(),
15334        );
15335        state.seeds = vec![refs::Seed {
15336            token: "magi/27b2/A".to_owned(),
15337            kind: refs::SeedKind::Unresolved,
15338            sha: String::new(),
15339            branch: true,
15340            detail: "no branch or commit named magi/27b2/A".to_owned(),
15341        }];
15342
15343        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15344        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15345        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15346        let detail = empty_candidate_detail(&state, "main");
15347        assert!(detail.starts_with("empty candidate"), "{detail}");
15348        assert!(detail.contains("magi/27b2/A"), "{detail}");
15349    }
15350
15351    /// `status == Ready` used to be read as "this is the harmless
15352    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15353    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15354    /// run's PR was closed without merging — and reentering `merge` with
15355    /// `mode` still `Pr` does not know the difference, so it pushed and
15356    /// opened a second pull request. `mode == Local` reproduces the same
15357    /// blind spot without a network call: reentry must not attempt another
15358    /// git merge once this node has already recorded an outcome.
15359    #[tokio::test]
15360    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15361        let tmp = tempfile::tempdir().expect("tempdir");
15362        let repo = tmp.path().join("repo");
15363        std::fs::create_dir_all(&repo).unwrap();
15364        init_repo(&repo);
15365
15366        let mut config = Config::default();
15367        config.merge.mode = MergeMode::Local;
15368
15369        let mut state = RunState::new(
15370            repo.clone(),
15371            "main".to_owned(),
15372            "deadbeef".to_owned(),
15373            "task".to_owned(),
15374            config,
15375        );
15376        state.candidates = vec![Candidate {
15377            index: 0,
15378            label: 'A',
15379            agent: "alpha".to_owned(),
15380            branch: "does-not-exist".to_owned(),
15381            worktree: repo.clone(),
15382            summary: String::new(),
15383            stat: String::new(),
15384            files: 0,
15385            commits: 0,
15386            empty: false,
15387            failed: None,
15388            verified_noop: None,
15389            duration_ms: 0,
15390            folded: false,
15391        }];
15392        state.tally = Some(Tally {
15393            first_choice: BTreeMap::from([('A', 1)]),
15394            borda: BTreeMap::new(),
15395            winner: 'A',
15396            rankings: 1,
15397            unanimous_initial: true,
15398            deliberated: false,
15399            changed_votes: 0,
15400            unanimous_final: true,
15401            tie_break: None,
15402            judges: 0,
15403            present: 0,
15404            quorum: 0,
15405            met_quorum: true,
15406            uncontested: Some("only candidate A produced a change".to_owned()),
15407        });
15408        state.reviews = vec![ReviewRound {
15409            round: 1,
15410            head: "deadbeef".to_owned(),
15411            verified_head: None,
15412            verified_at: None,
15413            reviews: Vec::new(),
15414            e2e: Vec::new(),
15415            fix: None,
15416            blocking: 0,
15417            answered: 0,
15418            expected: 0,
15419            clean: true,
15420            verify_retried: false,
15421            e2e_deferred: false,
15422            e2e_defer_reason: None,
15423            progressed: false,
15424            vote_split: false,
15425            reconsideration: Vec::new(),
15426            verdict: None,
15427        }];
15428        state.gate = vec![CommandOutcome {
15429            command: "test".to_owned(),
15430            code: Some(0),
15431            output_tail: String::new(),
15432            duration_ms: 0,
15433            resource_blocked: false,
15434        }];
15435        state.gate_ran = true;
15436        // Reached its conclusion already — e.g. `land` closing the PR without
15437        // merging it, which (like the honest `MergeMode::None` path) leaves
15438        // `status` at `Ready`. The recorded outcome is what actually marks
15439        // this node done.
15440        state.status = RunStatus::Ready;
15441        state.merge = Some(MergeOutcome {
15442            mode: MergeMode::Local,
15443            ok: false,
15444            detail: "already concluded".to_owned(),
15445            empty: false,
15446        });
15447
15448        let mut runner = Runner {
15449            state,
15450            roles: ResolvedRoles {
15451                implementers: Vec::new(),
15452                judges: Vec::new(),
15453                reviewers: Vec::new(),
15454                fixer: None,
15455                conductor: conductor(),
15456                implementer_roster: Vec::new(),
15457                judge_roster: Vec::new(),
15458                reviewer_roster: Vec::new(),
15459            },
15460            sem: Arc::new(Semaphore::new(1)),
15461            pause: Pause::new(),
15462            interrupt: Pause::new(),
15463        };
15464
15465        runner.merge().await.expect("merge");
15466
15467        assert_eq!(
15468            runner.state.status,
15469            RunStatus::Ready,
15470            "a concluded run's status must not change on reentry"
15471        );
15472        assert_eq!(
15473            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15474            Some("already concluded"),
15475            "merge must not run again once the node already recorded an outcome"
15476        );
15477    }
15478
15479    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15480    /// when its last attempt was resource-blocked (the shared build cache
15481    /// could not be acquired or confirmed fresh in time - see
15482    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15483    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15484    /// a run merge a tree the gate never actually checked - exactly the case
15485    /// a contended cache produces on every retry until it clears. `merge`
15486    /// must refuse until `gate` has actually recorded an attempt.
15487    #[tokio::test]
15488    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15489        let tmp = tempfile::tempdir().expect("tempdir");
15490        let repo = tmp.path().join("repo");
15491        std::fs::create_dir_all(&repo).unwrap();
15492        init_repo(&repo);
15493
15494        let mut config = Config::default();
15495        config.merge.mode = MergeMode::Local;
15496
15497        let mut state = RunState::new(
15498            repo.clone(),
15499            "main".to_owned(),
15500            "deadbeef".to_owned(),
15501            "task".to_owned(),
15502            config,
15503        );
15504        state.candidates = vec![Candidate {
15505            index: 0,
15506            label: 'A',
15507            agent: "alpha".to_owned(),
15508            branch: "does-not-exist".to_owned(),
15509            worktree: repo.clone(),
15510            summary: String::new(),
15511            stat: String::new(),
15512            files: 0,
15513            commits: 0,
15514            empty: false,
15515            failed: None,
15516            verified_noop: None,
15517            duration_ms: 0,
15518            folded: false,
15519        }];
15520        state.tally = Some(Tally {
15521            first_choice: BTreeMap::from([('A', 1)]),
15522            borda: BTreeMap::new(),
15523            winner: 'A',
15524            rankings: 1,
15525            unanimous_initial: true,
15526            deliberated: false,
15527            changed_votes: 0,
15528            unanimous_final: true,
15529            tie_break: None,
15530            judges: 0,
15531            present: 0,
15532            quorum: 0,
15533            met_quorum: true,
15534            uncontested: Some("only candidate A produced a change".to_owned()),
15535        });
15536        state.reviews = vec![ReviewRound {
15537            round: 1,
15538            head: "deadbeef".to_owned(),
15539            verified_head: None,
15540            verified_at: None,
15541            reviews: Vec::new(),
15542            e2e: Vec::new(),
15543            fix: None,
15544            blocking: 0,
15545            answered: 0,
15546            expected: 0,
15547            clean: true,
15548            verify_retried: false,
15549            e2e_deferred: false,
15550            e2e_defer_reason: None,
15551            progressed: false,
15552            vote_split: false,
15553            reconsideration: Vec::new(),
15554            verdict: None,
15555        }];
15556        // The point: `gate` has not recorded anything yet.
15557        state.gate = Vec::new();
15558        state.gate_ran = false;
15559        state.status = RunStatus::Gating;
15560
15561        let mut runner = Runner {
15562            state,
15563            roles: ResolvedRoles {
15564                implementers: Vec::new(),
15565                judges: Vec::new(),
15566                reviewers: Vec::new(),
15567                fixer: None,
15568                conductor: conductor(),
15569                implementer_roster: Vec::new(),
15570                judge_roster: Vec::new(),
15571                reviewer_roster: Vec::new(),
15572            },
15573            sem: Arc::new(Semaphore::new(1)),
15574            pause: Pause::new(),
15575            interrupt: Pause::new(),
15576        };
15577
15578        runner.merge().await.expect("merge");
15579
15580        assert!(
15581            runner.state.merge.is_none(),
15582            "an empty gate must never be read as a passing one: {:?}",
15583            runner.state.merge
15584        );
15585    }
15586
15587    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
15588    /// commands configured and `merge.mode` is `none` (a review-only run).
15589    /// `gate` must still record a real attempt — zero commands, vacuously
15590    /// passed — rather than leaving `state.gate` empty in a way `merge`
15591    /// cannot tell apart from "never ran"; otherwise the run reaches
15592    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
15593    #[tokio::test]
15594    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15595        let tmp = tempfile::tempdir().expect("tempdir");
15596        let repo = tmp.path().join("repo");
15597        std::fs::create_dir_all(&repo).unwrap();
15598        init_repo(&repo);
15599
15600        // Default config: `verify.gate` empty, `merge.mode` is `none`.
15601        let config = Config::default();
15602
15603        let mut state = RunState::new(
15604            repo.clone(),
15605            "main".to_owned(),
15606            "deadbeef".to_owned(),
15607            "task".to_owned(),
15608            config,
15609        );
15610        state.candidates = vec![Candidate {
15611            index: 0,
15612            label: 'A',
15613            agent: "alpha".to_owned(),
15614            branch: "does-not-exist".to_owned(),
15615            worktree: repo.clone(),
15616            summary: String::new(),
15617            stat: String::new(),
15618            files: 0,
15619            commits: 0,
15620            empty: false,
15621            failed: None,
15622            verified_noop: None,
15623            duration_ms: 0,
15624            folded: false,
15625        }];
15626        state.tally = Some(Tally {
15627            first_choice: BTreeMap::from([('A', 1)]),
15628            borda: BTreeMap::new(),
15629            winner: 'A',
15630            rankings: 1,
15631            unanimous_initial: true,
15632            deliberated: false,
15633            changed_votes: 0,
15634            unanimous_final: true,
15635            tie_break: None,
15636            judges: 0,
15637            present: 0,
15638            quorum: 0,
15639            met_quorum: true,
15640            uncontested: Some("only candidate A produced a change".to_owned()),
15641        });
15642        state.reviews = vec![ReviewRound {
15643            round: 1,
15644            head: "deadbeef".to_owned(),
15645            verified_head: None,
15646            verified_at: None,
15647            reviews: Vec::new(),
15648            e2e: Vec::new(),
15649            fix: None,
15650            blocking: 0,
15651            answered: 0,
15652            expected: 0,
15653            clean: true,
15654            verify_retried: false,
15655            e2e_deferred: false,
15656            e2e_defer_reason: None,
15657            progressed: false,
15658            vote_split: false,
15659            reconsideration: Vec::new(),
15660            verdict: None,
15661        }];
15662
15663        let mut runner = Runner {
15664            state,
15665            roles: ResolvedRoles {
15666                implementers: Vec::new(),
15667                judges: Vec::new(),
15668                reviewers: Vec::new(),
15669                fixer: None,
15670                conductor: conductor(),
15671                implementer_roster: Vec::new(),
15672                judge_roster: Vec::new(),
15673                reviewer_roster: Vec::new(),
15674            },
15675            sem: Arc::new(Semaphore::new(1)),
15676            pause: Pause::new(),
15677            interrupt: Pause::new(),
15678        };
15679
15680        runner.gate().await.expect("gate");
15681        assert!(
15682            runner.state.gate_ran,
15683            "zero configured commands is still a real attempt, not an unrun gate"
15684        );
15685        assert!(runner.state.gate.is_empty());
15686        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15687        assert_ne!(
15688            runner.state.status,
15689            RunStatus::Blocked,
15690            "a gate with nothing to check must not read as failed"
15691        );
15692
15693        runner.merge().await.expect("merge");
15694        assert_eq!(
15695            runner.state.status,
15696            RunStatus::Ready,
15697            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15698        );
15699    }
15700
15701    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15702    /// `verify.gate` (so the e2e leg and the final gate never build against
15703    /// different directories). With zero `verify.gate` commands but a
15704    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15705    /// that lease before discovering it had nothing to run - so a repo with
15706    /// no gate commands could come back `resource_blocked` (and therefore
15707    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15708    /// cache, exactly the contention this run's own zero commands could
15709    /// never have touched. `gate` must recognise there is nothing to check
15710    /// before it ever asks for the lease.
15711    #[tokio::test]
15712    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15713        crate::run::pin_test_home();
15714        let home = crate::run::home();
15715
15716        let tmp = tempfile::tempdir().expect("tempdir");
15717        let repo = tmp.path().join("repo");
15718        std::fs::create_dir_all(&repo).unwrap();
15719        init_repo(&repo);
15720        // Unique to this test, so holding its lease cannot collide with
15721        // another test sharing the same process-wide `home`.
15722        let cache_dir = tmp.path().join("target");
15723
15724        let mut config = Config::default();
15725        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15726        // `verify.gate` stays empty (the default). Bounded so a regression
15727        // that does start waiting fails the test in seconds, not hangs it.
15728        config.graph.timeout_verify = Some(2);
15729        config.graph.timeout_resource_wait = 2;
15730
15731        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15732        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15733            .expect("no io error acquiring directly")
15734        {
15735            crate::cache::AcquireOutcome::Acquired(g) => g,
15736            crate::cache::AcquireOutcome::Busy(b) => {
15737                panic!("expected the direct acquire to win the lease first: {b:?}")
15738            }
15739        };
15740
15741        let mut state = RunState::new(
15742            repo.clone(),
15743            "main".to_owned(),
15744            "deadbeef".to_owned(),
15745            "task".to_owned(),
15746            config,
15747        );
15748        state.candidates = vec![Candidate {
15749            index: 0,
15750            label: 'A',
15751            agent: "alpha".to_owned(),
15752            branch: "does-not-exist".to_owned(),
15753            worktree: repo.clone(),
15754            summary: String::new(),
15755            stat: String::new(),
15756            files: 0,
15757            commits: 0,
15758            empty: false,
15759            failed: None,
15760            verified_noop: None,
15761            duration_ms: 0,
15762            folded: false,
15763        }];
15764        state.tally = Some(Tally {
15765            first_choice: BTreeMap::from([('A', 1)]),
15766            borda: BTreeMap::new(),
15767            winner: 'A',
15768            rankings: 1,
15769            unanimous_initial: true,
15770            deliberated: false,
15771            changed_votes: 0,
15772            unanimous_final: true,
15773            tie_break: None,
15774            judges: 0,
15775            present: 0,
15776            quorum: 0,
15777            met_quorum: true,
15778            uncontested: Some("only candidate A produced a change".to_owned()),
15779        });
15780        state.reviews = vec![ReviewRound {
15781            round: 1,
15782            head: "deadbeef".to_owned(),
15783            verified_head: None,
15784            verified_at: None,
15785            reviews: Vec::new(),
15786            e2e: Vec::new(),
15787            fix: None,
15788            blocking: 0,
15789            answered: 0,
15790            expected: 0,
15791            clean: true,
15792            verify_retried: false,
15793            e2e_deferred: false,
15794            e2e_defer_reason: None,
15795            progressed: false,
15796            vote_split: false,
15797            reconsideration: Vec::new(),
15798            verdict: None,
15799        }];
15800
15801        let mut runner = Runner {
15802            state,
15803            roles: ResolvedRoles {
15804                implementers: Vec::new(),
15805                judges: Vec::new(),
15806                reviewers: Vec::new(),
15807                fixer: None,
15808                conductor: conductor(),
15809                implementer_roster: Vec::new(),
15810                judge_roster: Vec::new(),
15811                reviewer_roster: Vec::new(),
15812            },
15813            sem: Arc::new(Semaphore::new(1)),
15814            pause: Pause::new(),
15815            interrupt: Pause::new(),
15816        };
15817
15818        let started = std::time::Instant::now();
15819        runner.gate().await.expect("gate");
15820        assert!(
15821            started.elapsed() < Duration::from_secs(1),
15822            "a gate with nothing to run must never wait on a lease it never needed"
15823        );
15824        assert!(
15825            runner.state.gate_ran,
15826            "zero commands is still a real, immediate attempt"
15827        );
15828        assert!(runner.state.gate.is_empty());
15829        assert_ne!(
15830            runner.state.status,
15831            RunStatus::Blocked,
15832            "must not read as resource-blocked on a lease it never asked for"
15833        );
15834    }
15835
15836    #[cfg(unix)]
15837    #[tokio::test]
15838    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15839        crate::run::pin_test_home();
15840        let tmp = tempfile::tempdir().unwrap();
15841        let mut config = Config::default();
15842        config.disk.min_free_bytes = 0;
15843        let mut state = RunState::new(
15844            tmp.path().to_owned(),
15845            "main".into(),
15846            "base".into(),
15847            "verification evidence".into(),
15848            config,
15849        );
15850        let (outcomes, pids) = run_commands(
15851            &mut state,
15852            "gate",
15853            "gate",
15854            1,
15855            &["sh".into(), "-c".into()],
15856            &[
15857                "printf before-timeout; exec sleep 30".into(),
15858                "kill -TERM $$".into(),
15859                "exit 7".into(),
15860                "printf success".into(),
15861            ],
15862            tmp.path(),
15863            Duration::from_secs(1),
15864        )
15865        .await;
15866        wait_for_timed_out_children_to_die(&pids).await;
15867        assert_eq!(
15868            pids.len(),
15869            1,
15870            "unleased timeout retains its death-check PID"
15871        );
15872        assert_eq!(outcomes.len(), 4);
15873        assert_eq!(outcomes[0].code, None);
15874        assert!(outcomes[0].output_tail.contains("before-timeout"));
15875        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15876        assert_eq!(outcomes[1].code, None);
15877        assert!(outcomes[1].output_tail.contains("command terminated:"));
15878        assert!(!outcomes[1].output_tail.contains("timed out"));
15879        assert_eq!(outcomes[2].code, Some(7));
15880        assert!(!outcomes[2].output_tail.contains("timed out"));
15881        assert_eq!(outcomes[3].code, Some(0));
15882        assert_eq!(outcomes[3].output_tail, "success");
15883        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15884    }
15885
15886    /// The addendum's second gap: a `verify.gate` command running for real
15887    /// wall-clock time had nothing at all to show for it in `active` before
15888    /// `run_commands` learned to record it — a run could sit in `Gating` for
15889    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
15890    /// what was actually happening. Proven with a genuinely still-running
15891    /// command, not just a before/after check on the final state: a poller
15892    /// task reads the same `run.json` `gate()` is writing, the same way the
15893    /// phone or `magi show` would, while the shell command is still blocked
15894    /// on its own release marker.
15895    #[tokio::test]
15896    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15897        crate::run::pin_test_home();
15898
15899        let tmp = tempfile::tempdir().expect("tempdir");
15900        let repo = tmp.path().join("repo");
15901        std::fs::create_dir_all(&repo).unwrap();
15902        init_repo(&repo);
15903
15904        let mut config = Config::default();
15905        config.verify.gate = vec![
15906            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15907             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15908                .to_owned(),
15909        ];
15910
15911        let mut state = RunState::new(
15912            repo.clone(),
15913            "main".to_owned(),
15914            "deadbeef".to_owned(),
15915            "task".to_owned(),
15916            config,
15917        );
15918        let run_id = state.id.clone();
15919        state.candidates = vec![Candidate {
15920            index: 0,
15921            label: 'A',
15922            agent: "alpha".to_owned(),
15923            branch: "does-not-exist".to_owned(),
15924            worktree: repo.clone(),
15925            summary: String::new(),
15926            stat: String::new(),
15927            files: 0,
15928            commits: 0,
15929            empty: false,
15930            failed: None,
15931            verified_noop: None,
15932            duration_ms: 0,
15933            folded: false,
15934        }];
15935        state.tally = Some(Tally {
15936            first_choice: BTreeMap::from([('A', 1)]),
15937            borda: BTreeMap::new(),
15938            winner: 'A',
15939            rankings: 1,
15940            unanimous_initial: true,
15941            deliberated: false,
15942            changed_votes: 0,
15943            unanimous_final: true,
15944            tie_break: None,
15945            judges: 0,
15946            present: 0,
15947            quorum: 0,
15948            met_quorum: true,
15949            uncontested: Some("only candidate A produced a change".to_owned()),
15950        });
15951        state.reviews = vec![ReviewRound {
15952            round: 1,
15953            head: "deadbeef".to_owned(),
15954            verified_head: None,
15955            verified_at: None,
15956            reviews: Vec::new(),
15957            e2e: Vec::new(),
15958            fix: None,
15959            blocking: 0,
15960            answered: 0,
15961            expected: 0,
15962            clean: true,
15963            verify_retried: false,
15964            e2e_deferred: false,
15965            e2e_defer_reason: None,
15966            progressed: false,
15967            vote_split: false,
15968            reconsideration: Vec::new(),
15969            verdict: None,
15970        }];
15971
15972        let mut runner = Runner {
15973            state,
15974            roles: ResolvedRoles {
15975                implementers: Vec::new(),
15976                judges: Vec::new(),
15977                reviewers: Vec::new(),
15978                fixer: None,
15979                conductor: conductor(),
15980                implementer_roster: Vec::new(),
15981                judge_roster: Vec::new(),
15982                reviewer_roster: Vec::new(),
15983            },
15984            sem: Arc::new(Semaphore::new(1)),
15985            pause: Pause::new(),
15986            interrupt: Pause::new(),
15987        };
15988
15989        let started_marker = repo.join("started.marker");
15990        let release_marker = repo.join("release.marker");
15991        let poller = tokio::spawn(async move {
15992            // Bounded so a regression that never records the task entry
15993            // fails this test in seconds instead of hanging the suite —
15994            // the same shape `a_park_requested_while_a_seat_is_mid_call_
15995            // does_not_cut_it_short` uses for the same reason.
15996            for _ in 0..100 {
15997                if started_marker.exists()
15998                    && let Ok(s) = crate::run::RunState::load(&run_id)
15999                    && let Some(a) = s.active.get("gate")
16000                {
16001                    std::fs::write(&release_marker, b"go").expect("release marker");
16002                    return Some(a.clone());
16003                }
16004                tokio::time::sleep(Duration::from_millis(50)).await;
16005            }
16006            None
16007        });
16008
16009        runner.gate().await.expect("gate");
16010        let captured = poller.await.expect("poller task");
16011        let captured = captured.expect(
16012            "the poller never saw a `gate` task entry in run.json while the command was \
16013             still blocked on its own release marker",
16014        );
16015
16016        assert_eq!(captured.task.as_deref(), Some("gate"));
16017        assert_eq!(captured.node, "gate");
16018        assert_eq!(captured.index, Some(1));
16019        assert_eq!(captured.total, Some(1));
16020        assert!(
16021            captured
16022                .command
16023                .as_deref()
16024                .is_some_and(|c| c.contains("started.marker")),
16025            "{captured:?}"
16026        );
16027
16028        assert!(
16029            runner.state.active.is_empty(),
16030            "the entry must be cleared once the command actually finished: {:?}",
16031            runner.state.active
16032        );
16033        assert!(runner.state.gate_ran);
16034        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16035    }
16036
16037    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16038    /// record for `land`; one with only a Minor, or no reject, leaves none.
16039    #[tokio::test]
16040    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16041        use crate::verdict::{Finding, ReviewVote, Severity};
16042        crate::run::pin_test_home();
16043        let tmp = tempfile::tempdir().expect("tempdir");
16044        let repo = tmp.path().join("repo");
16045        std::fs::create_dir_all(&repo).unwrap();
16046        init_repo(&repo);
16047
16048        for (severity, vote, expect) in [
16049            (Severity::Major, ReviewVote::Reject, true),
16050            (Severity::Minor, ReviewVote::Reject, false),
16051            (Severity::Major, ReviewVote::Approve, false),
16052        ] {
16053            let mut round = review_round(false, 1, 1, 1, false, true);
16054            round.reviews = vec![ReviewRecord {
16055                reviewer: 1,
16056                agent: "alpha".to_owned(),
16057                summary: String::new(),
16058                findings: vec![Finding {
16059                    id: "R1-1-1".to_owned(),
16060                    severity,
16061                    file: None,
16062                    line: None,
16063                    title: "t".to_owned(),
16064                    detail: String::new(),
16065                }],
16066                vote: Some(vote),
16067                failed: None,
16068                duration_ms: 0,
16069                attempts: 0,
16070            }];
16071            let mut state = RunState::new(
16072                repo.clone(),
16073                "main".to_owned(),
16074                "deadbeef".to_owned(),
16075                "task".to_owned(),
16076                Config::default(),
16077            );
16078            state.reviews = vec![round];
16079            let mut runner = Runner {
16080                state,
16081                roles: ResolvedRoles {
16082                    implementers: Vec::new(),
16083                    judges: Vec::new(),
16084                    reviewers: Vec::new(),
16085                    fixer: None,
16086                    conductor: conductor(),
16087                    implementer_roster: Vec::new(),
16088                    judge_roster: Vec::new(),
16089                    reviewer_roster: Vec::new(),
16090                },
16091                sem: Arc::new(Semaphore::new(1)),
16092                pause: Pause::new(),
16093                interrupt: Pause::new(),
16094            };
16095            let shell = runner.state.config.shell();
16096            runner
16097                .stop_reviewing("round budget spent", &shell, &repo)
16098                .await
16099                .expect("stop_reviewing");
16100            assert_eq!(runner.state.status, RunStatus::Gating);
16101            assert_eq!(
16102                runner.state.contested_handoff.is_some(),
16103                expect,
16104                "{severity:?} + {vote:?}"
16105            );
16106        }
16107    }
16108
16109    /// The shape the incident this whole fix responds to actually had: the
16110    /// round budget spent, the last round's own e2e blocked on the shared
16111    /// build cache (held here by a live pid — this test process — exactly
16112    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16113    /// without forking a process). `stop_reviewing` must retry it — not
16114    /// silently leave the round looking untouched (the catch-up-only half of
16115    /// the bug), and not read the contention as a red `e2e` and block the
16116    /// run on it (the other half). Called directly, the same way
16117    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16118    /// above exercises `gate`, so this never needs a real cargo build to
16119    /// reach: the lease is never released, so `with_cache_lease` never gets
16120    /// past acquiring it into anything that would need a real workspace.
16121    #[tokio::test]
16122    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16123        crate::run::pin_test_home();
16124        let home = crate::run::home();
16125
16126        let tmp = tempfile::tempdir().expect("tempdir");
16127        let repo = tmp.path().join("repo");
16128        std::fs::create_dir_all(&repo).unwrap();
16129        init_repo(&repo);
16130        let head = crate::git::rev_parse(&repo, "HEAD")
16131            .await
16132            .expect("rev-parse");
16133        // Unique to this test, so holding its lease cannot collide with
16134        // another test sharing the same process-wide `home`.
16135        let cache_dir = tmp.path().join("target");
16136
16137        let mut config = Config::default();
16138        config.verify.e2e = vec![format!(
16139            "CARGO_TARGET_DIR='{}' test -f README.md",
16140            cache_dir.display()
16141        )];
16142        config.graph.review_rounds = 1;
16143        // Bounded so a regression that does start waiting fails the test in
16144        // seconds, not hangs it.
16145        config.graph.timeout_verify = Some(2);
16146        config.graph.timeout_resource_wait = 2;
16147
16148        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16149        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16150            .expect("no io error acquiring directly")
16151        {
16152            crate::cache::AcquireOutcome::Acquired(g) => g,
16153            crate::cache::AcquireOutcome::Busy(b) => {
16154                panic!("expected the direct acquire to win the lease first: {b:?}")
16155            }
16156        };
16157
16158        let mut state = RunState::new(
16159            repo.clone(),
16160            "main".to_owned(),
16161            head.clone(),
16162            "task".to_owned(),
16163            config,
16164        );
16165        state.candidates = vec![Candidate {
16166            index: 0,
16167            label: 'A',
16168            agent: "alpha".to_owned(),
16169            branch: "does-not-exist".to_owned(),
16170            worktree: repo.clone(),
16171            summary: String::new(),
16172            stat: String::new(),
16173            files: 0,
16174            commits: 0,
16175            empty: false,
16176            failed: None,
16177            verified_noop: None,
16178            duration_ms: 0,
16179            folded: false,
16180        }];
16181        state.tally = Some(Tally {
16182            first_choice: BTreeMap::from([('A', 1)]),
16183            borda: BTreeMap::new(),
16184            winner: 'A',
16185            rankings: 1,
16186            unanimous_initial: true,
16187            deliberated: false,
16188            changed_votes: 0,
16189            unanimous_final: true,
16190            tie_break: None,
16191            judges: 0,
16192            present: 0,
16193            quorum: 0,
16194            met_quorum: true,
16195            uncontested: Some("only candidate A produced a change".to_owned()),
16196        });
16197        // The round budget's last round, deferred: `needs_catchup_run`'s
16198        // other trigger. `stop_reviewing`'s retry machinery must treat this
16199        // exactly like a resource-blocked attempt once it actually runs.
16200        state.reviews = vec![ReviewRound {
16201            round: 1,
16202            head: head.clone(),
16203            verified_head: None,
16204            verified_at: None,
16205            reviews: Vec::new(),
16206            e2e: Vec::new(),
16207            fix: None,
16208            blocking: 1,
16209            answered: 1,
16210            expected: 1,
16211            clean: false,
16212            verify_retried: false,
16213            e2e_deferred: true,
16214            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16215            progressed: false,
16216            vote_split: false,
16217            reconsideration: Vec::new(),
16218            verdict: None,
16219        }];
16220
16221        let mut runner = Runner {
16222            state,
16223            roles: ResolvedRoles {
16224                implementers: Vec::new(),
16225                judges: Vec::new(),
16226                reviewers: Vec::new(),
16227                fixer: None,
16228                conductor: conductor(),
16229                implementer_roster: Vec::new(),
16230                judge_roster: Vec::new(),
16231                reviewer_roster: Vec::new(),
16232            },
16233            sem: Arc::new(Semaphore::new(1)),
16234            pause: Pause::new(),
16235            interrupt: Pause::new(),
16236        };
16237
16238        let shell = runner.state.config.shell();
16239        runner
16240            .stop_reviewing("round budget spent", &shell, &repo)
16241            .await
16242            .expect("stop_reviewing");
16243
16244        let last = runner.state.reviews.last().expect("round record");
16245        assert_eq!(
16246            last.e2e_status(),
16247            E2eStatus::ResourceBlocked,
16248            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16249             failed: {last:?}"
16250        );
16251        assert_eq!(
16252            last.verified_head.as_deref(),
16253            Some(head.as_str()),
16254            "which commit this attempt targeted is known even though nothing finished checking \
16255             it"
16256        );
16257        let first_attempt_at = last
16258            .verified_at
16259            .expect("when this attempt ran is known too");
16260        assert_ne!(
16261            runner.state.status,
16262            RunStatus::Blocked,
16263            "contention is evidence about the machine, not the patch — it must not settle the \
16264             run as blocked: {:?}",
16265            runner.state.status
16266        );
16267        assert!(
16268            !runner
16269                .state
16270                .events
16271                .iter()
16272                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16273            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16274            runner.state.events
16275        );
16276
16277        // The cache is still held: a later reentry must retry the same
16278        // round's verification again — not leave it looking exactly as
16279        // untouched as the first blocked attempt, which is indistinguishable
16280        // from never having tried again at all.
16281        runner
16282            .stop_reviewing("round budget spent", &shell, &repo)
16283            .await
16284            .expect("stop_reviewing retry");
16285        assert_eq!(
16286            runner.state.reviews.len(),
16287            1,
16288            "no new round was started: {:?}",
16289            runner.state.reviews
16290        );
16291        let last = runner.state.reviews.last().expect("round record");
16292        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16293        assert!(
16294            last.verified_at.expect("still known") > first_attempt_at,
16295            "a second reentry must be a fresh attempt, not a stale copy of the first"
16296        );
16297        assert_ne!(runner.state.status, RunStatus::Blocked);
16298
16299        held.release();
16300    }
16301
16302    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
16303    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
16304    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
16305    /// own top-of-function fast path (`review_conclusion`) correctly reads
16306    /// this shape as `None` rather than guessing `Blocked`, and the loop's
16307    /// own `for` range is empty once the round budget is spent, so
16308    /// `review_loop` must retry the check itself rather than silently doing
16309    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
16310    /// above exercises directly, but through `review_loop`'s own entry point
16311    /// this time, proving the wiring between the two rather than just the
16312    /// retry logic in isolation.
16313    #[tokio::test]
16314    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16315        crate::run::pin_test_home();
16316        let home = crate::run::home();
16317
16318        let tmp = tempfile::tempdir().expect("tempdir");
16319        let repo = tmp.path().join("repo");
16320        std::fs::create_dir_all(&repo).unwrap();
16321        init_repo(&repo);
16322        let head = crate::git::rev_parse(&repo, "HEAD")
16323            .await
16324            .expect("rev-parse");
16325        let cache_dir = tmp.path().join("target");
16326
16327        let mut config = Config::default();
16328        config.verify.e2e = vec![format!(
16329            "CARGO_TARGET_DIR='{}' test -f README.md",
16330            cache_dir.display()
16331        )];
16332        config.graph.review_rounds = 1;
16333        config.graph.timeout_verify = Some(2);
16334        config.graph.timeout_resource_wait = 2;
16335
16336        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16337        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16338            .expect("no io error acquiring directly")
16339        {
16340            crate::cache::AcquireOutcome::Acquired(g) => g,
16341            crate::cache::AcquireOutcome::Busy(b) => {
16342                panic!("expected the direct acquire to win the lease first: {b:?}")
16343            }
16344        };
16345
16346        let mut state = RunState::new(
16347            repo.clone(),
16348            "main".to_owned(),
16349            head.clone(),
16350            "task".to_owned(),
16351            config,
16352        );
16353        state.candidates = vec![Candidate {
16354            index: 0,
16355            label: 'A',
16356            agent: "alpha".to_owned(),
16357            branch: "does-not-exist".to_owned(),
16358            worktree: repo.clone(),
16359            summary: String::new(),
16360            stat: String::new(),
16361            files: 0,
16362            commits: 0,
16363            empty: false,
16364            failed: None,
16365            verified_noop: None,
16366            duration_ms: 0,
16367            folded: false,
16368        }];
16369        state.tally = Some(Tally {
16370            first_choice: BTreeMap::from([('A', 1)]),
16371            borda: BTreeMap::new(),
16372            winner: 'A',
16373            rankings: 1,
16374            unanimous_initial: true,
16375            deliberated: false,
16376            changed_votes: 0,
16377            unanimous_final: true,
16378            tie_break: None,
16379            judges: 0,
16380            present: 0,
16381            quorum: 0,
16382            met_quorum: true,
16383            uncontested: Some("only candidate A produced a change".to_owned()),
16384        });
16385        // The exact shape a prior process's `stop_reviewing` would have left
16386        // on disk: the round budget's last round, a real attempt already
16387        // made and already resource-blocked.
16388        state.reviews = vec![ReviewRound {
16389            round: 1,
16390            head: head.clone(),
16391            verified_head: Some(head.clone()),
16392            verified_at: Some(jiff::Timestamp::now()),
16393            reviews: Vec::new(),
16394            e2e: vec![CommandOutcome {
16395                command: format!(
16396                    "CARGO_TARGET_DIR='{}' test -f README.md",
16397                    cache_dir.display()
16398                ),
16399                code: None,
16400                output_tail: "waiting for the shared build cache".to_owned(),
16401                duration_ms: 0,
16402                resource_blocked: true,
16403            }],
16404            fix: None,
16405            blocking: 1,
16406            answered: 1,
16407            expected: 1,
16408            clean: false,
16409            verify_retried: false,
16410            e2e_deferred: false,
16411            e2e_defer_reason: None,
16412            progressed: false,
16413            vote_split: false,
16414            reconsideration: Vec::new(),
16415            verdict: None,
16416        }];
16417
16418        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16419        let mut runner = Runner {
16420            state,
16421            roles: ResolvedRoles {
16422                implementers: Vec::new(),
16423                judges: Vec::new(),
16424                reviewers: Vec::new(),
16425                fixer: None,
16426                conductor: conductor(),
16427                implementer_roster: Vec::new(),
16428                judge_roster: Vec::new(),
16429                reviewer_roster: Vec::new(),
16430            },
16431            sem: Arc::new(Semaphore::new(1)),
16432            pause: Pause::new(),
16433            interrupt: Pause::new(),
16434        };
16435
16436        // The lease is still held throughout, so this reentry's own retry is
16437        // also contended — proving `review_loop` actually tried again (not
16438        // that it happened to succeed) is what the timestamp comparison
16439        // below is for.
16440        runner.review_loop().await.expect("review_loop");
16441
16442        assert_eq!(
16443            runner.state.reviews.len(),
16444            1,
16445            "no new round was started on top of the unresolved one: {:?}",
16446            runner.state.reviews
16447        );
16448        let last = &runner.state.reviews[0];
16449        assert_eq!(
16450            last.e2e_status(),
16451            E2eStatus::ResourceBlocked,
16452            "still contended: {last:?}"
16453        );
16454        assert!(
16455            last.verified_at.expect("still known") > first_attempt_at,
16456            "review_loop must have actually retried the check, not left it exactly as found"
16457        );
16458        assert_ne!(
16459            runner.state.status,
16460            RunStatus::Blocked,
16461            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16462            runner.state.status
16463        );
16464
16465        held.release();
16466    }
16467
16468    #[tokio::test]
16469    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16470        crate::run::pin_test_home();
16471        for status in [RunStatus::Blocked, RunStatus::Gating] {
16472            let mut runner = runner_at(status);
16473            runner.state.config.graph.land = true;
16474            runner.state.gate_ran = true;
16475            runner.state.merge = Some(MergeOutcome {
16476                mode: MergeMode::Pr,
16477                ok: false,
16478                detail: "the fixer produced no commit".into(),
16479                empty: false,
16480            });
16481            let url = "https://example.invalid/o/r/pull/1";
16482            runner.state.pr = Some(crate::run::PrRecord {
16483                url: url.into(),
16484                number: 1,
16485                state: "open".into(),
16486                checks: "red".into(),
16487                round: 1,
16488                rounds: 1,
16489                red_at_merge: Vec::new(),
16490            });
16491            runner.state.save().unwrap();
16492            // The nonexistent repository makes observation fail before a forge
16493            // command can run. We still must reach land, rather than silently
16494            // return from the graph with its old gate and merge records.
16495            runner.execute_graph().await.unwrap();
16496            assert_eq!(runner.state.status, RunStatus::Blocked);
16497            assert!(runner.state.land_sync_pending);
16498            assert!(runner.state.events.iter().any(|event|
16499                event.node == "land" && event.message == format!("watching {url}")));
16500            assert_eq!(
16501                runner.state.merge.as_ref().unwrap().detail,
16502                "the fixer produced no commit"
16503            );
16504        }
16505    }
16506
16507    #[tokio::test]
16508    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16509        crate::run::pin_test_home();
16510        let tmp = tempfile::tempdir().expect("tempdir");
16511        let repo = tmp.path().join("repo");
16512        std::fs::create_dir_all(&repo).unwrap();
16513        init_repo(&repo);
16514
16515        let mut config = Config::default();
16516        config.merge.mode = MergeMode::Pr;
16517        config.graph.land = true;
16518        config.graph.land_approval = false;
16519
16520        let mut state = RunState::new(
16521            repo.clone(),
16522            "main".to_owned(),
16523            "deadbeef".to_owned(),
16524            "task".to_owned(),
16525            config,
16526        );
16527        state.candidates = vec![Candidate {
16528            index: 0,
16529            label: 'A',
16530            agent: "alpha".to_owned(),
16531            branch: "does-not-exist".to_owned(),
16532            worktree: repo.clone(),
16533            summary: String::new(),
16534            stat: String::new(),
16535            files: 0,
16536            commits: 0,
16537            empty: false,
16538            failed: None,
16539            verified_noop: None,
16540            duration_ms: 0,
16541            folded: false,
16542        }];
16543        state.tally = Some(Tally {
16544            first_choice: BTreeMap::from([('A', 1)]),
16545            borda: BTreeMap::new(),
16546            winner: 'A',
16547            rankings: 1,
16548            unanimous_initial: true,
16549            deliberated: false,
16550            changed_votes: 0,
16551            unanimous_final: true,
16552            tie_break: None,
16553            judges: 0,
16554            present: 0,
16555            quorum: 0,
16556            met_quorum: true,
16557            uncontested: Some("only candidate A produced a change".to_owned()),
16558        });
16559        state.reviews = vec![ReviewRound {
16560            round: 1,
16561            head: "deadbeef".to_owned(),
16562            verified_head: None,
16563            verified_at: None,
16564            reviews: Vec::new(),
16565            e2e: Vec::new(),
16566            fix: None,
16567            blocking: 0,
16568            answered: 0,
16569            expected: 0,
16570            clean: true,
16571            verify_retried: false,
16572            e2e_deferred: false,
16573            e2e_defer_reason: None,
16574            progressed: false,
16575            vote_split: false,
16576            reconsideration: Vec::new(),
16577            verdict: None,
16578        }];
16579        state.gate = vec![CommandOutcome {
16580            command: "test".to_owned(),
16581            code: Some(0),
16582            output_tail: String::new(),
16583            duration_ms: 0,
16584            resource_blocked: false,
16585        }];
16586        state.gate_ran = true;
16587        // A first pass through `merge` already pushed and opened this pull
16588        // request; `status` is `Landing` because a previous call into `land`
16589        // parked or was interrupted before it reached a terminal outcome.
16590        state.status = RunStatus::Landing;
16591        state.merge = Some(MergeOutcome {
16592            mode: MergeMode::Pr,
16593            ok: true,
16594            detail: "https://example.invalid/x/y/pull/1".to_owned(),
16595            empty: false,
16596        });
16597
16598        // The Landing-resume shortcut calls `run_land` directly rather than
16599        // through `merge`, which is exactly the call site that used to skip
16600        // `settle_questions` - see the fixture below.
16601        ask_test_home();
16602        // A real resume loads a persisted run; driver ownership is checked
16603        // against that record before the Landing shortcut is entered.
16604        state.save().expect("save landing run");
16605        let store = ask::Questions::open();
16606        let q = ask_open_question(&store, &state.id);
16607
16608        let mut runner = Runner {
16609            state,
16610            roles: ResolvedRoles {
16611                implementers: Vec::new(),
16612                judges: Vec::new(),
16613                reviewers: Vec::new(),
16614                fixer: None,
16615                conductor: conductor(),
16616                implementer_roster: Vec::new(),
16617                judge_roster: Vec::new(),
16618                reviewer_roster: Vec::new(),
16619            },
16620            sem: Arc::new(Semaphore::new(1)),
16621            pause: Pause::new(),
16622            interrupt: Pause::new(),
16623        };
16624
16625        // `execute`, not `merge` directly: the Landing-resume shortcut lives
16626        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
16627        // exactly because `review_loop` would otherwise clobber the marker
16628        // first.
16629        runner.execute().await.expect("execute");
16630
16631        assert_eq!(
16632            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16633            Some("https://example.invalid/x/y/pull/1"),
16634            "reentry must not push again or open a second pull request over the \
16635             one `land` is already watching"
16636        );
16637        assert_ne!(
16638            runner.state.status,
16639            RunStatus::Landing,
16640            "land could not actually reach the fake pull request, so it must \
16641             have given up rather than left the run silently parked forever"
16642        );
16643        // `land` could not reach the fake pull request, so it gave up into
16644        // `Blocked` - still resumable, so the question must not have been
16645        // swept just because this branch now also calls `settle_questions`.
16646        assert_eq!(runner.state.status, RunStatus::Blocked);
16647        assert!(
16648            store.get(&q.id).unwrap().status.open(),
16649            "Blocked is still alive; settle_questions must have been a no-op here"
16650        );
16651    }
16652
16653    fn state_with_round(round: ReviewRound) -> RunState {
16654        let mut s = RunState::new(
16655            PathBuf::from("/repo"),
16656            "main".to_owned(),
16657            "abc1234".to_owned(),
16658            "add retries".to_owned(),
16659            Config::default(),
16660        );
16661        s.reviews = vec![round];
16662        s
16663    }
16664
16665    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16666        crate::verdict::Finding {
16667            id: id.to_owned(),
16668            severity,
16669            file: None,
16670            line: None,
16671            title: title.to_owned(),
16672            detail: String::new(),
16673        }
16674    }
16675
16676    #[test]
16677    fn pr_body_names_open_findings_and_declined_ones() {
16678        let round = ReviewRound {
16679            round: 2,
16680            head: "deadbee".to_owned(),
16681            verified_head: None,
16682            verified_at: None,
16683            reviews: vec![ReviewRecord {
16684                attempts: 0,
16685                reviewer: 1,
16686                agent: "alpha".to_owned(),
16687                summary: String::new(),
16688                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16689                vote: None,
16690                failed: None,
16691                duration_ms: 0,
16692            }],
16693            e2e: vec![CommandOutcome {
16694                command: "cargo test".to_owned(),
16695                code: Some(0),
16696                output_tail: String::new(),
16697                duration_ms: 0,
16698                resource_blocked: false,
16699            }],
16700            verify_retried: false,
16701            e2e_deferred: false,
16702            e2e_defer_reason: None,
16703            fix: Some(FixRecord {
16704                agent: "alpha".to_owned(),
16705                addressed: Vec::new(),
16706                rejected: vec![crate::verdict::Rejection {
16707                    id: "R1-1-1".to_owned(),
16708                    why: "not reachable from any caller".to_owned(),
16709                }],
16710                notes: String::new(),
16711                committed: true,
16712                failed: None,
16713                duration_ms: 0,
16714                continuation: None,
16715            }),
16716            blocking: 0,
16717            answered: 1,
16718            expected: 1,
16719            clean: false,
16720            progressed: true,
16721            vote_split: false,
16722            reconsideration: Vec::new(),
16723            verdict: None,
16724        };
16725        let state = state_with_round(round);
16726        let body = pr_message(&state, 'A').body;
16727
16728        assert!(body.contains("add retries"), "the task must still be there");
16729        assert!(body.contains("R2-1-1"), "{body}");
16730        assert!(!body.contains("unused import"), "{body}");
16731        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16732        assert!(
16733            !body.contains("not reachable from any caller"),
16734            "the reason it was declined: {body}"
16735        );
16736    }
16737
16738    #[test]
16739    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16740        let round = ReviewRound {
16741            round: 1,
16742            head: "deadbee".to_owned(),
16743            verified_head: None,
16744            verified_at: None,
16745            reviews: vec![ReviewRecord {
16746                attempts: 0,
16747                reviewer: 1,
16748                agent: "alpha".to_owned(),
16749                summary: String::new(),
16750                findings: Vec::new(),
16751                vote: None,
16752                failed: None,
16753                duration_ms: 0,
16754            }],
16755            e2e: Vec::new(),
16756            verify_retried: false,
16757            e2e_deferred: false,
16758            e2e_defer_reason: None,
16759            fix: None,
16760            blocking: 0,
16761            answered: 1,
16762            expected: 1,
16763            clean: true,
16764            progressed: false,
16765            vote_split: false,
16766            reconsideration: Vec::new(),
16767            verdict: None,
16768        };
16769        let state = state_with_round(round);
16770        let body = pr_message(&state, 'A').body;
16771        assert!(!body.contains("Open review findings"), "{body}");
16772        assert!(!body.contains("Declined"), "{body}");
16773    }
16774
16775    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16776        let mut state = RunState::new(
16777            PathBuf::from("/repo"),
16778            "main".to_owned(),
16779            "abc1234".to_owned(),
16780            instruction.to_owned(),
16781            Config::default(),
16782        );
16783        state.candidates.push(Candidate {
16784            index: 0,
16785            label: 'A',
16786            agent: "alpha".to_owned(),
16787            branch: "magi/x/A".to_owned(),
16788            worktree: PathBuf::from("/wt"),
16789            summary: summary.to_owned(),
16790            stat: String::new(),
16791            files: 1,
16792            commits: 1,
16793            empty: false,
16794            failed: None,
16795            verified_noop: None,
16796            folded: false,
16797            duration_ms: 0,
16798        });
16799        state
16800    }
16801
16802    fn review_state(subjects: &[&str]) -> RunState {
16803        let mut state = state_with_summary(
16804            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16805            "",
16806        );
16807        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16808        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16809        state
16810    }
16811
16812    /// A branch rebased onto a main that moved past the recorded
16813    /// `base_commit` is titled from its own first commit, never main's.
16814    #[tokio::test]
16815    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16816        ask_test_home();
16817        let tmp = tempfile::tempdir().unwrap();
16818        let repo = tmp.path().join("repo");
16819        std::fs::create_dir_all(&repo).unwrap();
16820        init_repo(&repo);
16821        let origin = tmp.path().join("origin.git");
16822        let g = |dir: &Path, args: &[&str]| {
16823            let out = std::process::Command::new("git")
16824                .args(args)
16825                .current_dir(dir)
16826                .quiet()
16827                .output()
16828                .expect("spawn git");
16829            assert!(
16830                out.status.success(),
16831                "git {args:?}: {}",
16832                String::from_utf8_lossy(&out.stderr)
16833            );
16834        };
16835        g(
16836            tmp.path(),
16837            &[
16838                "clone",
16839                "--bare",
16840                "-q",
16841                repo.to_str().unwrap(),
16842                origin.to_str().unwrap(),
16843            ],
16844        );
16845        g(
16846            &repo,
16847            &["remote", "add", "origin", origin.to_str().unwrap()],
16848        );
16849        let c1 = git::rev_parse(&repo, "main").await.unwrap();
16850
16851        // Main moves on; the branch is built on top of the new main.
16852        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16853        g(&repo, &["add", "-A"]);
16854        g(
16855            &repo,
16856            &["commit", "-q", "-m", "chore(deps): update a crate"],
16857        );
16858        g(&repo, &["push", "-q", "origin", "main"]);
16859        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16860        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16861        g(&repo, &["add", "-A"]);
16862        g(
16863            &repo,
16864            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16865        );
16866        g(&repo, &["checkout", "-q", "main"]);
16867
16868        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16869        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16870        // Without a readable tracking ref the recorded base's merge base is used.
16871        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16872        assert_eq!(fallback, c1);
16873
16874        let mut state = review_state(&[
16875            "chore(deps): update a crate",
16876            "fix(daemon): apply a chosen action",
16877        ]);
16878        state.repo = repo.clone();
16879        state.base_branch = "main".to_owned();
16880        state.base_commit = c1;
16881        refresh_reviewed_commits(&mut state, "feat/own").await;
16882        assert_eq!(
16883            state.reviewed_commits,
16884            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16885        );
16886        assert_eq!(
16887            review_title(&state).as_deref(),
16888            Some("fix(daemon): apply a chosen action")
16889        );
16890        assert_eq!(
16891            leaked_subjects(&state, "feat/own").await,
16892            Some(vec!["chore(deps): update a crate".to_owned()])
16893        );
16894        // A stale tracking ref is still used when the fetch fails, but the
16895        // leak list is withheld.
16896        state.config.merge.remote = "nowhere".to_owned();
16897        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16898    }
16899
16900    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
16901    /// with `origin/main` one commit behind it, and a summarizing agent that
16902    /// prints whatever `reply` holds.
16903    struct SummaryFixture {
16904        _tmp: tempfile::TempDir,
16905        repo: PathBuf,
16906        reply: PathBuf,
16907        runner: Runner,
16908    }
16909
16910    fn summary_fixture() -> SummaryFixture {
16911        ask_test_home();
16912        let tmp = tempfile::tempdir().unwrap();
16913        let repo = tmp.path().join("repo");
16914        std::fs::create_dir_all(&repo).unwrap();
16915        init_repo(&repo);
16916        let origin = tmp.path().join("origin.git");
16917        let g = |args: &[&str]| {
16918            let out = std::process::Command::new("git")
16919                .args(args)
16920                .current_dir(&repo)
16921                .quiet()
16922                .output()
16923                .expect("spawn git");
16924            assert!(
16925                out.status.success(),
16926                "git {args:?}: {}",
16927                String::from_utf8_lossy(&out.stderr)
16928            );
16929        };
16930        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16931        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16932        g(&["push", "-q", "origin", "main"]);
16933        g(&["checkout", "-q", "-b", "feat/own"]);
16934        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16935        g(&["add", "-A"]);
16936        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16937        let reply = tmp.path().join("reply.txt");
16938        std::fs::write(
16939            &reply,
16940            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16941        )
16942        .unwrap();
16943        let mut runner = runner_at(RunStatus::Gating);
16944        let mut state = review_state(&["feat(own): add the own file"]);
16945        state.repo = repo.clone();
16946        state.base_branch = "main".to_owned();
16947        state.candidates[0].branch = "feat/own".to_owned();
16948        state.candidates[0].worktree = repo.clone();
16949        let mut author = spec("alpha");
16950        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16951        author
16952            .env
16953            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16954        state.config.agents = vec![author];
16955        runner.state = state;
16956        SummaryFixture {
16957            _tmp: tmp,
16958            repo,
16959            reply,
16960            runner,
16961        }
16962    }
16963
16964    async fn posted(f: &mut SummaryFixture) -> PrMessage {
16965        let winner = f.runner.state.candidates[0].clone();
16966        f.runner
16967            .guarded_pr_message(&winner, None, true)
16968            .await
16969            .unwrap()
16970            .expect("an English description never parks")
16971    }
16972
16973    fn clean_round_with_minor_findings() -> ReviewRound {
16974        let mut round = passing_round(2, None);
16975        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16976        round.reviews = vec![ReviewRecord {
16977            attempts: 0,
16978            reviewer: 1,
16979            agent: "alpha".into(),
16980            summary: "内部レビュー".into(),
16981            findings: vec![
16982                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16983                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16984            ],
16985            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16986            failed: None,
16987            duration_ms: 0,
16988        }];
16989        round
16990    }
16991
16992    #[tokio::test]
16993    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16994        for review_only in [false, true] {
16995            let mut f = summary_fixture();
16996            if !review_only {
16997                f.runner.state.candidates[0].agent = "alpha".into();
16998            }
16999            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17000            assert!(f.runner.state.reviews[0].clean);
17001            assert!(f.runner.state.reviews[0].fix.is_none());
17002            assert!(f.runner.state.open_findings().is_empty());
17003            assert_eq!(
17004                crate::pr_outcome::required_finding_ids(&f.runner.state),
17005                ["R2-1-1", "R2-1-2"]
17006            );
17007            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17008            let evidence = crate::pr_outcome::summary_evidence(
17009                &f.runner.state,
17010                crate::pr_outcome::FinalHead::At(&head),
17011            );
17012            assert!(
17013                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17014            );
17015            // The ordinary reply omits both IDs: no cached description is adopted.
17016            let omitted = posted(&mut f).await;
17017            assert!(omitted.body.contains("No description"));
17018            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17019            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17020            assert!(f.runner.state.change_summary.is_none());
17021            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17022            let described = posted(&mut f).await;
17023            assert!(described.body.contains("improve documentation for callers"));
17024            assert!(described.body.contains("remaining edge case"));
17025            assert!(described.body.contains("## Open review findings"));
17026            assert!(!described.body.contains("未修正"));
17027        }
17028    }
17029
17030    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17031        let mut f = summary_fixture();
17032        if !review_only {
17033            f.runner.state.candidates[0].agent = "alpha".into();
17034        }
17035        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17036        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17037        fixed
17038            .fix
17039            .as_mut()
17040            .unwrap()
17041            .rejected
17042            .push(crate::verdict::Rejection {
17043                id: "R1-1-1".into(),
17044                why: "呼び出されないコードのため辞退".into(),
17045            });
17046        let mut clean = clean_round_with_minor_findings();
17047        clean.verified_head = Some(head.clone());
17048        f.runner.state.reviews = vec![fixed, clean];
17049        ok_gate(&mut f.runner.state, Some(&head));
17050        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17051        f.runner.state.config.agents.push(scripted("beta", reply));
17052        f.runner.state.config.roles.language_judge =
17053            Some(crate::config::AgentChoice::One("beta".into()));
17054        f
17055    }
17056
17057    #[tokio::test]
17058    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17059        for review_only in [false, true] {
17060            let mut f = rewrite_fixture_with_findings(review_only,
17061                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17062            let m = posted(&mut f).await;
17063            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17064            let expected =
17065                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17066            assert!(m.body.contains(&expected));
17067            assert!(m.body.contains("## Review rounds"));
17068            assert!(
17069                m.body
17070                    .contains("Gate (`verify.gate`): passed on the final commit")
17071            );
17072            assert!(
17073                m.body
17074                    .contains("Tests (`verify.e2e`): passed on the final commit")
17075            );
17076            let c = f.runner.state.change_summary.as_ref().unwrap();
17077            assert!(
17078                m.body
17079                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17080            );
17081            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17082            assert!(crate::pr_outcome::covers_required_findings(
17083                &f.runner.state,
17084                &m.body
17085            ));
17086            assert!(m.body.contains("documentation remains incomplete"));
17087            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17088            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17089            assert!(f.runner.state.github_text.is_none());
17090        }
17091    }
17092
17093    #[tokio::test]
17094    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17095        for review_only in [false, true] {
17096            for reply in [
17097                "TITLE: fix: public description\nA short English description.",
17098                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17099            ] {
17100                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17101                let winner = f.runner.state.candidates[0].clone();
17102                let posted = f
17103                    .runner
17104                    .guarded_pr_message(&winner, None, true)
17105                    .await
17106                    .unwrap();
17107                assert!(posted.is_none());
17108                assert!(f.runner.state.parked);
17109                let withheld = f.runner.state.github_text.as_ref().unwrap();
17110                assert!(
17111                    withheld.rewrite.is_none(),
17112                    "an incomplete rewrite must not be selectable"
17113                );
17114                let q = ask::Questions::open()
17115                    .get(withheld.question.as_ref().unwrap())
17116                    .unwrap();
17117                assert!(
17118                    !q.choices
17119                        .contains(&crate::github_text::USE_REWRITE.to_owned())
17120                );
17121                assert!(
17122                    q.detail.contains("R1-1-1")
17123                        && q.detail.contains("R2-1-1")
17124                        && q.detail.contains("R2-1-2")
17125                );
17126            }
17127        }
17128    }
17129
17130    #[tokio::test]
17131    async fn japanese_reports_stay_local_in_both_public_run_modes() {
17132        for review_only in [false, true] {
17133            let mut f = summary_fixture();
17134            if !review_only {
17135                f.runner.state.candidates[0].agent = "alpha".into();
17136            }
17137            f.runner.state.instruction = "内部タスクの説明".into();
17138            f.runner.state.candidates[0].summary = "内部の候補報告".into();
17139            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17140            f.runner.state.reviews = vec![
17141                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17142                round_with_notes(2, Some("二回目の内部修正報告です。")),
17143                passing_round(3, Some(&head)),
17144            ];
17145            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17146            ok_gate(&mut f.runner.state, Some(&head));
17147            let captured = f.repo.join("public-prompt.txt");
17148            let author = &mut f.runner.state.config.agents[0];
17149            author.command = vec![
17150                "sh".into(),
17151                "-c".into(),
17152                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17153            ];
17154            author.env.insert(
17155                "PROMPT_FILE".into(),
17156                captured.to_string_lossy().into_owned(),
17157            );
17158            let m = posted(&mut f).await;
17159            assert!(m.body.contains("Adds `own.txt`"));
17160            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17161            assert!(m.body.contains("R1-1-1"));
17162            assert!(
17163                m.body
17164                    .contains("Gate (`verify.gate`): passed on the final commit")
17165            );
17166            assert!(
17167                m.body
17168                    .contains("Tests (`verify.e2e`): passed on the final commit")
17169            );
17170            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17171            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17172            let prompt = std::fs::read_to_string(captured).unwrap();
17173            assert!(prompt.contains(&format!(
17174                "git diff {} {head}",
17175                f.runner.state.change_summary.as_ref().unwrap().base
17176            )));
17177            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17178            assert_eq!(
17179                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17180                "二回目の内部修正報告です。"
17181            );
17182        }
17183    }
17184
17185    #[tokio::test]
17186    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17187        for review_only in [false, true] {
17188            let mut f = summary_fixture();
17189            if !review_only {
17190                f.runner.state.candidates[0].agent = "alpha".into();
17191            }
17192            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17193            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17194            std::fs::write(&f.reply, "no usable description").unwrap();
17195            let failed = posted(&mut f).await;
17196            assert!(
17197                failed
17198                    .body
17199                    .contains("No description of what this change does could be generated")
17200            );
17201            assert!(failed.body.is_ascii());
17202            assert!(f.runner.state.github_text.is_none());
17203            assert!(f.runner.state.change_summary.is_none());
17204            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17205            let evidence = crate::pr_outcome::summary_evidence(
17206                &f.runner.state,
17207                crate::pr_outcome::FinalHead::At(&head),
17208            );
17209            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17210            assert!(
17211                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17212                    .exists()
17213            );
17214            f.runner.state =
17215                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17216            std::fs::write(
17217                &f.reply,
17218                "TITLE: feat(own): add the own file\nAdds the committed own file.",
17219            )
17220            .unwrap();
17221            let retried = posted(&mut f).await;
17222            assert!(retried.body.contains("Adds the committed own file."));
17223            assert_eq!(
17224                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17225                "日本語の修正報告"
17226            );
17227        }
17228    }
17229
17230    #[tokio::test]
17231    async fn same_head_verification_and_findings_invalidate_public_prose() {
17232        let mut f = summary_fixture();
17233        posted(&mut f).await;
17234        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17235        let original = f
17236            .runner
17237            .state
17238            .change_summary
17239            .as_ref()
17240            .unwrap()
17241            .evidence
17242            .clone();
17243        ok_gate(&mut f.runner.state, Some(&head));
17244        std::fs::write(
17245            &f.reply,
17246            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17247        )
17248        .unwrap();
17249        let verified = posted(&mut f).await;
17250        assert!(verified.body.contains("now verified"));
17251        assert_ne!(
17252            f.runner.state.change_summary.as_ref().unwrap().evidence,
17253            original
17254        );
17255        let mut round = round_with_notes(1, Some("内部報告"));
17256        round.reviews.push(ReviewRecord {
17257            attempts: 0,
17258            reviewer: 1,
17259            agent: "alpha".into(),
17260            summary: "内部レビュー".into(),
17261            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17262            vote: None,
17263            failed: None,
17264            duration_ms: 0,
17265        });
17266        round
17267            .fix
17268            .as_mut()
17269            .unwrap()
17270            .rejected
17271            .push(crate::verdict::Rejection {
17272                id: "R1-1-1".into(),
17273                why: "呼び出し元から到達できない".into(),
17274            });
17275        f.runner.state.reviews = vec![round];
17276        // A description omitting the required ID is unusable, even with a title.
17277        let omitted = posted(&mut f).await;
17278        assert!(omitted.body.contains("No description"));
17279        assert!(omitted.body.contains("R1-1-1"));
17280        assert!(!omitted.body.contains("到達"));
17281        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17282        let described = posted(&mut f).await;
17283        assert!(described.body.contains("unreachable from callers"));
17284        assert!(described.body.contains("R1-1-1"));
17285        assert!(!described.body.contains("到達"));
17286        let evidence = f
17287            .runner
17288            .state
17289            .change_summary
17290            .as_ref()
17291            .unwrap()
17292            .evidence
17293            .clone();
17294        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17295        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17296        assert!(posted(&mut f).await.body.contains("rationale updated"));
17297        assert_ne!(
17298            f.runner.state.change_summary.as_ref().unwrap().evidence,
17299            evidence
17300        );
17301        // Old serialized summaries have no evidence fingerprint and must regenerate.
17302        f.runner
17303            .state
17304            .change_summary
17305            .as_mut()
17306            .unwrap()
17307            .evidence
17308            .clear();
17309        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17310        assert!(posted(&mut f).await.body.contains("current specification"));
17311    }
17312
17313    #[tokio::test]
17314    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17315        let mut f = summary_fixture();
17316        let m = posted(&mut f).await;
17317        assert_eq!(m.title, "feat(own): add the own file");
17318        assert!(
17319            m.body
17320                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17321            "{}",
17322            m.body
17323        );
17324        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17325        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17326        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17327        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17328        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17329        let kept = f.runner.state.change_summary.as_ref().unwrap();
17330        assert_eq!(
17331            (kept.head.as_str(), kept.base.as_str()),
17332            (head.as_str(), base.as_str())
17333        );
17334        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17335    }
17336
17337    #[tokio::test]
17338    async fn review_only_summary_receives_the_configured_github_language() {
17339        let mut f = summary_fixture();
17340        f.runner.state.config.graph.github_language = "ja".into();
17341        let captured = f.repo.join("summary-prompt.txt");
17342        let author = &mut f.runner.state.config.agents[0];
17343        author.command = vec![
17344            "sh".into(),
17345            "-c".into(),
17346            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17347        ];
17348        author.env.insert(
17349            "PROMPT_FILE".into(),
17350            captured.to_string_lossy().into_owned(),
17351        );
17352        std::fs::write(
17353            &f.reply,
17354            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17355        )
17356        .unwrap();
17357        let m = posted(&mut f).await;
17358        let prompt = std::fs::read_to_string(captured).unwrap();
17359        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17360        assert!(!prompt.contains("in English"));
17361        assert_eq!(m.title, "feat(own): ファイルを追加");
17362        assert!(m.body.contains("own.txt を追加します。"));
17363    }
17364
17365    #[tokio::test]
17366    async fn review_fixes_come_after_the_whole_change_summary() {
17367        let mut f = summary_fixture();
17368        f.runner.state.reviews = vec![
17369            round_with_notes(1, Some("handled the PATH case")),
17370            round_with_notes(2, None),
17371        ];
17372        let m = posted(&mut f).await;
17373        let summary = m.body.find("## Summary").expect("summary");
17374        let fixes = m.body.find("## Review rounds").expect("review rounds");
17375        assert!(summary < fixes, "{}", m.body);
17376        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17377        assert!(m.body.contains("## Final verification"), "{}", m.body);
17378    }
17379
17380    #[tokio::test]
17381    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17382        let mut f = summary_fixture();
17383        let first = posted(&mut f).await;
17384        let first_head = f.runner.state.change_summary.clone().unwrap().head;
17385        // The same head reuses the record: a different reply is not read.
17386        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17387        let again = posted(&mut f).await;
17388        assert_eq!((again.title, again.body), (first.title, first.body));
17389
17390        let g = |args: &[&str]| {
17391            let out = std::process::Command::new("git")
17392                .args(args)
17393                .current_dir(&f.repo)
17394                .quiet()
17395                .output()
17396                .expect("spawn git");
17397            assert!(out.status.success(), "git {args:?}");
17398        };
17399        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17400        g(&["add", "-A"]);
17401        g(&["commit", "-q", "-m", "fix(own): handle more"]);
17402        let second = posted(&mut f).await;
17403        let kept = f.runner.state.change_summary.clone().unwrap();
17404        assert_ne!(kept.head, first_head);
17405        assert!(second.body.contains("Not read yet."), "{}", second.body);
17406        assert!(!second.body.contains("so the first summary is read"));
17407        assert_eq!(second.title, "feat(own): other");
17408    }
17409
17410    #[tokio::test]
17411    async fn an_unavailable_summary_is_stated_and_never_invented() {
17412        let mut f = summary_fixture();
17413        std::fs::write(&f.reply, "no title line here").unwrap();
17414        let m = posted(&mut f).await;
17415        assert!(
17416            m.body
17417                .contains("No description of what this change does could be generated"),
17418            "{}",
17419            m.body
17420        );
17421        assert!(m.body.contains("internal reports remain in local records"));
17422        assert!(f.runner.state.change_summary.is_none());
17423        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17424        // A preview asks nobody and claims nothing.
17425        let winner = f.runner.state.candidates[0].clone();
17426        let preview = f
17427            .runner
17428            .guarded_pr_message(&winner, None, false)
17429            .await
17430            .unwrap()
17431            .unwrap();
17432        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17433    }
17434
17435    #[test]
17436    fn pr_message_review_single_commit_uses_its_subject() {
17437        let state = review_state(&["feat(nats): per-role user"]);
17438        let m = pr_message(&state, 'A');
17439        assert_eq!(m.title, "feat(nats): per-role user");
17440        assert!(!m.body.contains("Review the work already"), "{}", m.body);
17441        assert!(m.body.contains("## Commits under review"), "{}", m.body);
17442    }
17443
17444    #[test]
17445    fn pr_message_review_multi_commit_takes_the_oldest() {
17446        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17447        let m = pr_message(&state, 'A');
17448        assert_eq!(m.title, "feat: the change");
17449        for s in ["feat: the change", "fix: typo", "fix: again"] {
17450            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17451        }
17452    }
17453
17454    #[test]
17455    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17456        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17457            let mut state = review_state(&[first, "fix: later fixup"]);
17458            state.candidates[0].branch = "機能/ログイン".to_owned();
17459            let m = pr_message(&state, 'A');
17460            assert!(
17461                m.title.starts_with("chore: land candidate A of run"),
17462                "{}",
17463                m.title
17464            );
17465        }
17466    }
17467
17468    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17469        BranchFacts {
17470            commits: commits
17471                .iter()
17472                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17473                .collect(),
17474            stat: stat.to_owned(),
17475        }
17476    }
17477
17478    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17479        let mut r = review_round(true, 0, 1, 1, true, true);
17480        r.round = round;
17481        r.fix = notes.map(|n| FixRecord {
17482            agent: "fixer".to_owned(),
17483            addressed: Vec::new(),
17484            rejected: Vec::new(),
17485            notes: n.to_owned(),
17486            committed: true,
17487            failed: None,
17488            duration_ms: 0,
17489            continuation: None,
17490        });
17491        r
17492    }
17493
17494    #[test]
17495    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17496        let state = review_state(&["ignored"]);
17497        let f = facts(
17498            &[
17499                (
17500                    "fix(login): resolve PATH on macOS",
17501                    "Login shells skip rc files.",
17502                ),
17503                ("fix: address review", ""),
17504            ],
17505            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17506        );
17507        let m = pr_message_with(&state, 'A', Some(&f));
17508        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17509        assert!(
17510            m.body
17511                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
17512            "{}",
17513            m.body
17514        );
17515        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17516        assert!(m.body.contains("## Diff stat"), "{}", m.body);
17517        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17518        for banned in [
17519            "Review the work already",
17520            "no task statement",
17521            "Original task",
17522        ] {
17523            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17524        }
17525        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17526    }
17527
17528    #[test]
17529    fn pr_message_review_truncates_a_huge_first_commit_body() {
17530        let state = review_state(&["ignored"]);
17531        let f = facts(
17532            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17533            "s",
17534        );
17535        let m = pr_message_with(&state, 'A', Some(&f));
17536        assert!(m.body.len() < 30_000, "{}", m.body.len());
17537        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17538        assert!(
17539            m.body.contains("- ... 1 more commit(s)"),
17540            "{}",
17541            m.body.len()
17542        );
17543        assert!(m.body.ends_with("magi:candidate-a\n"));
17544    }
17545
17546    #[test]
17547    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17548        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17549        assert_eq!(m.title, "feat: x");
17550        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17551        assert!(!m.body.contains("Diff stat"), "{}", m.body);
17552        assert!(!m.body.contains("no task statement"), "{}", m.body);
17553    }
17554
17555    #[test]
17556    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17557        let mut state = review_state(&["日本語の件名"]);
17558        state.candidates[0].branch = "fix/macos-login-path".to_owned();
17559        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17560        state.candidates[0].branch = "機能/ログイン".to_owned();
17561        assert!(
17562            pr_message(&state, 'A')
17563                .title
17564                .starts_with("chore: land candidate A")
17565        );
17566    }
17567
17568    #[test]
17569    fn pr_message_review_fixes_survive_a_clean_final_round() {
17570        let mut state = review_state(&["feat: x"]);
17571        state.reviews = vec![
17572            round_with_notes(1, Some("handled the PATH case")),
17573            round_with_notes(2, None),
17574        ];
17575        let body = pr_message(&state, 'A').body;
17576        assert!(
17577            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17578            "{body}"
17579        );
17580        assert!(body.contains("### Round 1"), "{body}");
17581
17582        state.reviews = vec![
17583            round_with_notes(1, Some("first fix")),
17584            round_with_notes(2, Some("")),
17585            round_with_notes(3, Some("second fix")),
17586            round_with_notes(4, None),
17587        ];
17588        let body = pr_message(&state, 'A').body;
17589        assert!(
17590            body.contains("### Round 1") && !body.contains("first fix"),
17591            "{body}"
17592        );
17593        assert!(
17594            body.contains("### Round 3") && !body.contains("second fix"),
17595            "{body}"
17596        );
17597        assert!(body.contains("### Round 2"), "{body}");
17598    }
17599
17600    const FINAL: &str = "abcdef1234567890";
17601
17602    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17603        let mut r = round_with_notes(round, None);
17604        r.verified_head = verified.map(str::to_owned);
17605        r
17606    }
17607
17608    fn ok_gate(state: &mut RunState, head: Option<&str>) {
17609        state.gate = vec![CommandOutcome {
17610            command: "gate".to_owned(),
17611            code: Some(0),
17612            output_tail: String::new(),
17613            duration_ms: 0,
17614            resource_blocked: false,
17615        }];
17616        state.gate_ran = true;
17617        state.gate_head = head.map(str::to_owned);
17618    }
17619
17620    fn body_at(state: &RunState) -> String {
17621        pr_message_raw(
17622            state,
17623            'A',
17624            None,
17625            crate::pr_outcome::FinalHead::At(FINAL),
17626            ChangeNote::NotAsked,
17627        )
17628        .body
17629    }
17630
17631    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17632
17633    #[test]
17634    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17635        let mut state = review_state(&["feat: x"]);
17636        state.reviews = vec![
17637            round_with_notes(1, Some(SANDBOXED)),
17638            passing_round(2, Some(FINAL)),
17639        ];
17640        ok_gate(&mut state, Some(FINAL));
17641        let body = body_at(&state);
17642        assert!(!body.contains("Handled the PATH case."), "{body}");
17643        assert!(!body.contains("could not commit"), "{body}");
17644        assert!(!body.contains("could not run"), "{body}");
17645        assert!(
17646            body.contains("The fix is committed on the branch"),
17647            "{body}"
17648        );
17649        assert!(!body.contains("sandbox"), "{body}");
17650        assert!(
17651            body.contains("Tests (`verify.e2e`): passed on the final commit"),
17652            "{body}"
17653        );
17654        assert!(
17655            body.contains("Gate (`verify.gate`): passed on the final commit"),
17656            "{body}"
17657        );
17658        // The raw report is untouched in the record.
17659        assert!(
17660            state.reviews[0]
17661                .fix
17662                .as_ref()
17663                .unwrap()
17664                .notes
17665                .contains("could not commit")
17666        );
17667    }
17668
17669    #[test]
17670    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17671        let mut state = review_state(&["feat: x"]);
17672        let mut r1 = round_with_notes(1, Some(SANDBOXED));
17673        r1.fix.as_mut().unwrap().committed = false;
17674        let mut r2 = passing_round(2, Some(FINAL));
17675        r2.e2e[0].code = Some(1);
17676        state.reviews = vec![r1, r2];
17677        let body = body_at(&state);
17678        assert!(!body.contains("could not commit"), "{body}");
17679        assert!(!body.contains("could not run"), "{body}");
17680        assert!(body.contains("This round produced no commit"), "{body}");
17681        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17682        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17683        assert!(!body.contains("passed"), "{body}");
17684    }
17685
17686    #[test]
17687    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17688        let mut state = review_state(&["feat: x"]);
17689        state.reviews = vec![
17690            round_with_notes(1, Some(SANDBOXED)),
17691            passing_round(2, Some("0123456789abcdef")),
17692        ];
17693        ok_gate(&mut state, Some("0123456789abcdef"));
17694        let body = body_at(&state);
17695        assert!(!body.contains("HTTP tests could not run"), "{body}");
17696        // The commit claim is still settled by the structured record.
17697        assert!(!body.contains("could not commit"), "{body}");
17698        assert!(
17699            body.contains("passed on an earlier commit (`0123456`)"),
17700            "{body}"
17701        );
17702        assert!(!body.contains("passed on the final commit"), "{body}");
17703
17704        // A gate recorded before `gate_head` existed is "not recorded".
17705        ok_gate(&mut state, None);
17706        let body = body_at(&state);
17707        assert!(
17708            body.contains("passed on a commit that was not recorded"),
17709            "{body}"
17710        );
17711    }
17712
17713    #[test]
17714    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17715        let state = state_with_summary(
17716            "add retries to the client",
17717            "TITLE: feat: retries\n\nDid it.",
17718        );
17719        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17720        assert_eq!(m.title, "feat: retries");
17721        assert!(
17722            m.body.contains("<summary>Original task</summary>"),
17723            "{}",
17724            m.body
17725        );
17726        assert!(!m.body.contains("Commits under review"), "{}", m.body);
17727        assert!(
17728            m.body
17729                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17730            "{}",
17731            m.body
17732        );
17733    }
17734
17735    #[test]
17736    fn pr_message_review_bounds_a_long_english_subject() {
17737        let long = format!("feat: {}", "word ".repeat(100));
17738        let m = pr_message(&review_state(&[&long]), 'A');
17739        assert!(m.title.starts_with("feat: word"), "{}", m.title);
17740        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17741    }
17742
17743    #[test]
17744    fn pr_message_implementation_run_is_unchanged_by_review_support() {
17745        let state = state_with_summary("add retries\n\ndetails", "- did some things");
17746        let m = pr_message(&state, 'A');
17747        assert_eq!(m.title, "add retries");
17748        assert!(m.body.contains("<summary>Original task</summary>"));
17749        assert!(!m.body.contains("Commits under review"));
17750        assert_eq!(landing_subject_source(&state), state.instruction);
17751    }
17752
17753    #[test]
17754    fn review_run_squash_subject_is_the_change_not_the_prompt() {
17755        let state = review_state(&["feat: the change", "fix: typo"]);
17756        let source = landing_subject_source(&state);
17757        assert_eq!(land::merge_subject("", &source), "feat: the change");
17758        assert_eq!(
17759            land::merge_subject("magi: candidate A (uncommitted work)", &source),
17760            "feat: the change"
17761        );
17762        // An operator's rename still wins.
17763        assert_eq!(
17764            land::merge_subject("feat: renamed by hand", &source),
17765            "feat: renamed by hand"
17766        );
17767        let blank = review_state(&["日本語"]);
17768        assert!(
17769            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17770        );
17771    }
17772
17773    #[test]
17774    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17775        let state = review_state(&["feat: the change"]);
17776        let source = landing_subject_source(&state);
17777        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17778        assert_eq!(
17779            land::merge_subject(landing_title(&state, old), &source),
17780            "feat: the change"
17781        );
17782        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17783        let task = state_with_summary("add retries", "");
17784        assert_eq!(landing_title(&task, old), old);
17785    }
17786
17787    #[test]
17788    fn pr_message_describes_the_change_not_the_task() {
17789        let state = state_with_summary(
17790            "今回やってほしいこと: results projector を直す",
17791            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17792        );
17793        let m = pr_message(&state, 'A');
17794        assert_eq!(m.title, "fix(web): batch the runs list reads");
17795        assert!(
17796            m.body.starts_with("## Summary\n\n- reads run.json once"),
17797            "{}",
17798            m.body
17799        );
17800        assert!(!m.body.contains("TITLE:"), "{}", m.body);
17801        let task_at = m.body.find("今回やってほしいこと").unwrap();
17802        let details_at = m.body.find("<details>").unwrap();
17803        assert!(
17804            details_at < task_at,
17805            "the task lives inside <details>: {}",
17806            m.body
17807        );
17808        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17809        assert!(m.body.contains("magi:candidate-a"));
17810    }
17811
17812    #[test]
17813    fn pr_message_falls_back_to_the_task_without_a_title_line() {
17814        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17815        let m = pr_message(&state, 'A');
17816        assert_eq!(m.title, "add retries");
17817        assert!(
17818            m.body.contains("## Summary\n\n- did some things"),
17819            "{}",
17820            m.body
17821        );
17822
17823        let none = RunState::new(
17824            PathBuf::from("/repo"),
17825            "main".to_owned(),
17826            "abc1234".to_owned(),
17827            "add retries".to_owned(),
17828            Config::default(),
17829        );
17830        let m = pr_message(&none, 'A');
17831        assert_eq!(m.title, "add retries");
17832        assert!(!m.body.contains("## Summary"), "{}", m.body);
17833    }
17834
17835    #[test]
17836    fn pr_message_refuses_the_candidate_commit_subject() {
17837        for bad in [
17838            "TITLE: magi: candidate A (uncommitted work)",
17839            "TITLE: chore: stuff (uncommitted work)",
17840            "TITLE:   ",
17841        ] {
17842            let state = state_with_summary("add retries", bad);
17843            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17844        }
17845    }
17846
17847    #[test]
17848    fn pr_message_bounds_a_very_long_task_and_title() {
17849        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17850        let state = state_with_summary(&long, "- nothing");
17851        let m = pr_message(&state, 'A');
17852        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17853        assert!(!m.title.contains('\n'));
17854
17855        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17856        let m = pr_message(&state, 'A');
17857        assert!(m.title.starts_with("feat: "));
17858        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17859        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17860    }
17861
17862    fn long_title_of(instruction: &str) -> String {
17863        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17864    }
17865
17866    #[test]
17867    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17868        let first = "Make the landing path keep a readable title for long tasks";
17869        let line = format!(
17870            "{first}. {}",
17871            "And then keep going with more words ".repeat(20)
17872        );
17873        let t = long_title_of(&line);
17874        assert_eq!(t, first);
17875        assert!(!t.starts_with("chore: land"));
17876    }
17877
17878    #[test]
17879    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17880        let line = "word ".repeat(200);
17881        let t = long_title_of(&line);
17882        assert!(t.ends_with("word..."), "{t}");
17883        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17884    }
17885
17886    #[test]
17887    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17888        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17889            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17890        }
17891    }
17892
17893    #[test]
17894    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17895        let mut state = state_with_summary("再試行を追加する", "- nothing");
17896        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17897        state.config.graph.github_language = "ja".into();
17898        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17899    }
17900
17901    #[test]
17902    fn pr_message_title_limit_is_exact() {
17903        let at = "a".repeat(PR_TITLE_MAX);
17904        assert_eq!(long_title_of(&at), at);
17905        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17906        assert!(over.ends_with("..."), "{over}");
17907        assert_eq!(over.chars().count(), PR_TITLE_MAX);
17908    }
17909
17910    #[test]
17911    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17912        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17913        let t = long_title_of(&line);
17914        assert!(t.ends_with("..."), "{t}");
17915        assert!(t.is_ascii(), "{t}");
17916    }
17917
17918    #[test]
17919    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17920        let line = format!(
17921            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17922            "plus more filler words ".repeat(20)
17923        );
17924        let t = long_title_of(&line);
17925        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17926    }
17927
17928    #[test]
17929    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17930        let line = format!("\n\n# {}", "title words ".repeat(40));
17931        let state = state_with_summary(&line, "- nothing");
17932        let m = pr_message(&state, 'A');
17933        assert!(m.title.starts_with("title words"), "{}", m.title);
17934        assert_eq!(
17935            land::merge_subject(&m.title, &landing_subject_source(&state)),
17936            m.title
17937        );
17938        // An operator's rename wins untouched.
17939        assert_eq!(
17940            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17941            "feat: renamed by hand"
17942        );
17943    }
17944
17945    #[tokio::test]
17946    async fn github_text_rewrite_is_bounded_and_falls_back() {
17947        crate::run::pin_test_home();
17948        for (reply, accepted) in [
17949            (
17950                "TITLE: fix: retries\nAdd retries for failed requests.",
17951                true,
17952            ),
17953            (
17954                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17955                false,
17956            ),
17957            (
17958                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17959                true,
17960            ),
17961            (
17962                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17963                true,
17964            ),
17965            (
17966                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17967                true,
17968            ),
17969            (
17970                "TITLE: fix: retries\n日本語の説明です。 token=secret",
17971                false,
17972            ),
17973            ("TITLE: fix: retries\n", false),
17974        ] {
17975            let dir = tempfile::tempdir().unwrap();
17976            let mut runner = runner_at(RunStatus::Gating);
17977            runner.state = state_with_summary(
17978                "add retries",
17979                "TITLE: fix: retries\n日本語の説明を書きます。",
17980            );
17981            runner.state.repo = dir.path().to_owned();
17982            runner.state.candidates[0].worktree = dir.path().to_owned();
17983            let mut author = spec("alpha");
17984            author.command = vec![
17985                "sh".into(),
17986                "-c".into(),
17987                "printf '%s' \"$REWRITE_REPLY\"".into(),
17988            ];
17989            author.env.insert("REWRITE_REPLY".into(), reply.into());
17990            runner.state.config.agents = vec![author];
17991            seed_public_summary(&mut runner);
17992            let winner = runner.state.candidates[0].clone();
17993            let first = runner
17994                .guarded_pr_message(&winner, None, true)
17995                .await
17996                .unwrap();
17997            let message = if accepted {
17998                first.expect("a passing rewrite never asks")
17999            } else {
18000                // Withheld: the run parks on a question and posts nothing yet.
18001                assert!(first.is_none());
18002                assert!(runner.state.parked);
18003                let store = ask::Questions::open();
18004                let asked: Vec<_> = store
18005                    .list()
18006                    .into_iter()
18007                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18008                    .collect();
18009                assert_eq!(asked.len(), 1);
18010                assert!(!asked[0].detail.contains("secret"));
18011                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18012                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18013                assert_eq!(
18014                    asked[0]
18015                        .choices
18016                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18017                    rewrite.is_some()
18018                );
18019                // Silence: the question lapses and the neutral text is used,
18020                // without a second rewrite call or a second question.
18021                store
18022                    .update(&asked[0].id, |q| {
18023                        q.abandon("test");
18024                        Ok(())
18025                    })
18026                    .unwrap();
18027                let again = runner
18028                    .guarded_pr_message(&winner, None, true)
18029                    .await
18030                    .unwrap()
18031                    .expect("an abandoned question falls back");
18032                let n = store
18033                    .list()
18034                    .into_iter()
18035                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18036                    .count();
18037                assert_eq!(n, 1, "asked once per rejected text");
18038                again
18039            };
18040            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18041            assert!(!message.body.contains("secret"));
18042            assert!(!message.body.contains("prefix"));
18043            assert!(!message.title.contains("secret"));
18044            assert_eq!(
18045                message.body.contains("Add retries for failed requests."),
18046                accepted
18047            );
18048            assert_eq!(
18049                runner.state.seats["github-text-rewrite"].turns, 1,
18050                "only one rewrite invocation"
18051            );
18052            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18053            if accepted {
18054                assert!(runner.state.github_text.is_none());
18055                assert!(!runner.state.parked);
18056                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18057                assert!(message.body.contains("## Final verification"));
18058                assert!(message.body.contains("Evidence: final diff"));
18059                assert!(
18060                    message
18061                        .body
18062                        .contains(&format!("magi:run/{}", runner.state.id))
18063                );
18064            }
18065            if !accepted {
18066                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18067                assert!(
18068                    message
18069                        .body
18070                        .contains(&format!("magi:run/{}", runner.state.id))
18071                );
18072            }
18073        }
18074    }
18075
18076    fn scripted(id: &str, reply: &str) -> AgentSpec {
18077        let mut a = spec(id);
18078        a.command = vec![
18079            "sh".into(),
18080            "-c".into(),
18081            "printf '%s' \"$REWRITE_REPLY\"".into(),
18082        ];
18083        a.env.insert("REWRITE_REPLY".into(), reply.into());
18084        a
18085    }
18086
18087    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18088    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18089
18090    /// Gate tests inject generated public output, not internal reports.
18091    fn seed_public_summary(runner: &mut Runner) {
18092        let repo = &runner.state.repo;
18093        let g = |args: &[&str]| {
18094            let out = std::process::Command::new("git")
18095                .args(args)
18096                .current_dir(repo)
18097                .quiet()
18098                .output()
18099                .unwrap();
18100            assert!(out.status.success(), "git {args:?}");
18101            String::from_utf8_lossy(&out.stdout).trim().to_owned()
18102        };
18103        g(&["init", "-q", "-b", "main"]);
18104        g(&[
18105            "-c",
18106            "user.name=Test",
18107            "-c",
18108            "user.email=test@example.invalid",
18109            "commit",
18110            "-q",
18111            "--allow-empty",
18112            "-m",
18113            "feat: fixture",
18114        ]);
18115        let head = g(&["rev-parse", "HEAD"]);
18116        g(&["update-ref", "refs/remotes/origin/main", &head]);
18117        let evidence = crate::pr_outcome::summary_evidence(
18118            &runner.state,
18119            crate::pr_outcome::FinalHead::At(&head),
18120        );
18121        let generated = &runner.state.candidates[0].summary;
18122        runner.state.change_summary = Some(crate::run::ChangeSummary {
18123            base: head.clone(),
18124            head,
18125            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18126            title: summary_title(generated).unwrap(),
18127            body: summary_without_title(generated),
18128        });
18129    }
18130
18131    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18132        let mut runner = runner_at(RunStatus::Gating);
18133        runner.state = state_with_summary("add retries", JA_SUMMARY);
18134        if review_only {
18135            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18136        }
18137        runner.state.repo = dir.to_owned();
18138        runner.state.candidates[0].worktree = dir.to_owned();
18139        runner.state.config.agents = agents;
18140        seed_public_summary(&mut runner);
18141        runner
18142    }
18143
18144    fn gate_questions(runner: &Runner) -> usize {
18145        ask::Questions::open()
18146            .list()
18147            .into_iter()
18148            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18149            .count()
18150    }
18151
18152    #[tokio::test]
18153    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18154        crate::run::pin_test_home();
18155        let dir = tempfile::tempdir().unwrap();
18156        let mut runner = rewrite_runner(
18157            dir.path(),
18158            true,
18159            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18160        );
18161        runner.state.config.roles.language_judge =
18162            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18163        let winner = runner.state.candidates[0].clone();
18164        let msg = runner
18165            .guarded_pr_message(&winner, None, true)
18166            .await
18167            .unwrap()
18168            .expect("an English rewrite posts without asking");
18169        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18170        assert!(msg.body.contains("<details>kept</details>"));
18171        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18172        assert_eq!(gate_questions(&runner), 0);
18173        assert!(runner.state.github_text.is_none());
18174        assert!(!runner.state.seats.contains_key("impl-A"));
18175        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18176    }
18177
18178    #[tokio::test]
18179    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18180        crate::run::pin_test_home();
18181        let dir = tempfile::tempdir().unwrap();
18182        let mut runner = rewrite_runner(
18183            dir.path(),
18184            false,
18185            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18186        );
18187        runner.state.config.roles.language_judge =
18188            Some(crate::config::AgentChoice::One("beta".into()));
18189        runner.state.seats.insert(
18190            "impl-A".into(),
18191            SeatState::new("impl-A", "alpha", runner.state.seed),
18192        );
18193        let winner = runner.state.candidates[0].clone();
18194        let msg = runner
18195            .guarded_pr_message(&winner, None, true)
18196            .await
18197            .unwrap()
18198            .expect("rewritten");
18199        assert!(msg.body.contains("<details>kept</details>"));
18200        assert_eq!(runner.state.seats["impl-A"].turns, 0);
18201        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18202        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18203    }
18204
18205    #[tokio::test]
18206    async fn rewrite_reuses_a_resumable_implementer_session() {
18207        crate::run::pin_test_home();
18208        let dir = tempfile::tempdir().unwrap();
18209        let mut runner = rewrite_runner(
18210            dir.path(),
18211            false,
18212            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18213        );
18214        runner.state.config.roles.language_judge =
18215            Some(crate::config::AgentChoice::One("beta".into()));
18216        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18217        seat.turns = 1;
18218        runner.state.seats.insert("impl-A".into(), seat);
18219        let winner = runner.state.candidates[0].clone();
18220        runner
18221            .guarded_pr_message(&winner, None, true)
18222            .await
18223            .unwrap()
18224            .expect("rewritten");
18225        assert_eq!(runner.state.seats["impl-A"].turns, 2);
18226        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18227    }
18228
18229    #[tokio::test]
18230    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18231        crate::run::pin_test_home();
18232        let dir = tempfile::tempdir().unwrap();
18233        let mut broken = spec("bad");
18234        broken.command = vec!["false".into()];
18235        let mut runner = rewrite_runner(
18236            dir.path(),
18237            true,
18238            vec![
18239                broken,
18240                scripted("noformat", "Add retries."),
18241                scripted("good", EN_REWRITE),
18242            ],
18243        );
18244        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18245            "bad".into(),
18246            "noformat".into(),
18247            "good".into(),
18248        ]));
18249        let winner = runner.state.candidates[0].clone();
18250        let msg = runner
18251            .guarded_pr_message(&winner, None, true)
18252            .await
18253            .unwrap()
18254            .expect("the third agent rewrites");
18255        assert!(msg.body.contains("`src/net.rs`"));
18256        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18257        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18258    }
18259
18260    #[tokio::test]
18261    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18262        crate::run::pin_test_home();
18263        // (agents, role): still Japanese; no TITLE; nothing resolvable.
18264        for (agents, role, saved) in [
18265            (
18266                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18267                Some("beta"),
18268                true,
18269            ),
18270            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18271            (Vec::new(), None, false),
18272        ] {
18273            let dir = tempfile::tempdir().unwrap();
18274            let mut runner = rewrite_runner(dir.path(), true, agents);
18275            runner.state.config.roles.language_judge =
18276                role.map(|r| crate::config::AgentChoice::One(r.into()));
18277            let winner = runner.state.candidates[0].clone();
18278            let first = runner
18279                .guarded_pr_message(&winner, None, true)
18280                .await
18281                .unwrap();
18282            assert!(first.is_none(), "withheld text parks the run");
18283            assert!(runner.state.parked);
18284            assert_eq!(gate_questions(&runner), 1);
18285            let g = runner.state.github_text.as_ref().unwrap();
18286            assert_eq!(g.rewrite.is_some(), saved);
18287            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18288        }
18289    }
18290
18291    #[tokio::test]
18292    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18293        crate::run::pin_test_home();
18294        let dir = tempfile::tempdir().unwrap();
18295        let mut runner = runner_at(RunStatus::Gating);
18296        runner.state =
18297            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18298        runner.state.repo = dir.path().to_owned();
18299        runner.state.candidates[0].worktree = dir.path().to_owned();
18300        let mut author = spec("alpha");
18301        author.command = vec![
18302            "sh".into(),
18303            "-c".into(),
18304            "printf '%s' \"$REWRITE_REPLY\"".into(),
18305        ];
18306        author.env.insert(
18307            "REWRITE_REPLY".into(),
18308            format!(
18309                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18310                "English sentence. ".repeat(600)
18311            ),
18312        );
18313        runner.state.config.agents = vec![author];
18314        seed_public_summary(&mut runner);
18315        let winner = runner.state.candidates[0].clone();
18316        assert!(
18317            runner
18318                .guarded_pr_message(&winner, None, true)
18319                .await
18320                .unwrap()
18321                .is_none()
18322        );
18323        let record = runner.state.github_text.clone().unwrap();
18324        let rewrite = record.rewrite.clone().unwrap();
18325        assert_eq!(rewrite.failed, ["body-language"]);
18326        assert!(
18327            rewrite
18328                .body
18329                .ends_with(&format!("magi:run/{}", runner.state.id))
18330        );
18331        let store = ask::Questions::open();
18332        let q = store.get(record.question.as_ref().unwrap()).unwrap();
18333        assert!(
18334            q.choices
18335                .contains(&crate::github_text::USE_REWRITE.to_owned())
18336        );
18337        for label in [
18338            "ORIGINAL",
18339            "FALLBACK",
18340            "REWRITE",
18341            "Rules still failed: body-language",
18342            "truncated",
18343        ] {
18344            assert!(q.detail.contains(label), "{label}");
18345        }
18346        let artifact = std::fs::read_to_string(crate::run::artifact_path(
18347            &runner.state,
18348            &crate::github_text::artifact_name(&record.fingerprint),
18349        ))
18350        .unwrap();
18351        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18352        // Restart from serialized state, then retry after a bad owner title.
18353        runner.state =
18354            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18355        store
18356            .update(&q.id, |q| {
18357                q.say("まだ日本語")?;
18358                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18359            })
18360            .unwrap();
18361        assert!(
18362            runner
18363                .guarded_pr_message(&winner, None, true)
18364                .await
18365                .unwrap()
18366                .is_none()
18367        );
18368        let retry = runner.state.github_text.as_ref().unwrap();
18369        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18370        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18371        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18372        assert!(
18373            q.posting_comparison
18374                .as_ref()
18375                .unwrap()
18376                .matches_detail(&q.detail)
18377        );
18378        store
18379            .update(&q.id, |q| {
18380                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18381            })
18382            .unwrap();
18383        let message = runner
18384            .guarded_pr_message(&winner, None, true)
18385            .await
18386            .unwrap()
18387            .unwrap();
18388        assert_eq!(message.title, rewrite.title);
18389        assert_eq!(message.body, rewrite.body);
18390        assert_eq!(
18391            runner.state.github_text.as_ref().unwrap().choice,
18392            Some(GithubTextChoice::Rewrite)
18393        );
18394        let again = runner
18395            .guarded_pr_message(&winner, None, true)
18396            .await
18397            .unwrap()
18398            .unwrap();
18399        assert_eq!(again.title, message.title);
18400        assert_eq!(again.body, message.body);
18401        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18402        assert!(
18403            !runner
18404                .state
18405                .events
18406                .iter()
18407                .any(|e| e.message.contains(&rewrite.title)
18408                    || e.message.contains("English sentence."))
18409        );
18410    }
18411
18412    #[tokio::test]
18413    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18414        crate::run::pin_test_home();
18415        let dir = tempfile::tempdir().unwrap();
18416        let mut runner = runner_at(RunStatus::Gating);
18417        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18418        runner.state.repo = dir.path().to_owned();
18419        runner.state.candidates[0].worktree = dir.path().to_owned();
18420        seed_public_summary(&mut runner);
18421        let winner = runner.state.candidates[0].clone();
18422        assert!(
18423            runner
18424                .guarded_pr_message(&winner, None, true)
18425                .await
18426                .unwrap()
18427                .is_none()
18428        );
18429        let record = runner.state.github_text.as_ref().unwrap();
18430        assert!(record.rewrite.is_none());
18431        // Records written before the additive field deserialize with no rewrite.
18432        let mut old = serde_json::to_value(record).unwrap();
18433        old.as_object_mut().unwrap().remove("rewrite");
18434        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18435        assert!(old.rewrite.is_none());
18436        let id = record.question.clone().unwrap();
18437        ask::Questions::open()
18438            .update(&id, |q| {
18439                q.choices.push(crate::github_text::USE_REWRITE.into());
18440                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18441            })
18442            .unwrap();
18443        let message = runner
18444            .guarded_pr_message(&winner, None, true)
18445            .await
18446            .unwrap()
18447            .unwrap();
18448        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18449        assert_eq!(
18450            runner.state.github_text.as_ref().unwrap().choice,
18451            Some(GithubTextChoice::Fallback)
18452        );
18453    }
18454
18455    #[tokio::test]
18456    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18457        crate::run::pin_test_home();
18458        let dir = tempfile::tempdir().unwrap();
18459        let mut runner = runner_at(RunStatus::Gating);
18460        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18461        runner.state.repo = dir.path().to_owned();
18462        runner.state.candidates[0].worktree = dir.path().to_owned();
18463        seed_public_summary(&mut runner);
18464        let winner = runner.state.candidates[0].clone();
18465        assert!(
18466            runner
18467                .guarded_pr_message(&winner, None, true)
18468                .await
18469                .unwrap()
18470                .is_none()
18471        );
18472        let store = ask::Questions::open();
18473        let find = |run: &str| {
18474            store
18475                .list()
18476                .into_iter()
18477                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18478                .unwrap()
18479        };
18480        let run = runner.state.id.clone();
18481        let q = find(&run);
18482        let fp = runner
18483            .state
18484            .github_text
18485            .as_ref()
18486            .unwrap()
18487            .fingerprint
18488            .clone();
18489        let name = crate::github_text::artifact_name(&fp);
18490        let path = crate::run::artifact_path(&runner.state, &name);
18491        assert!(q.detail.contains(&path.display().to_string()));
18492        assert!(q.detail.contains("title-language: title"));
18493        let saved = std::fs::read_to_string(&path).expect("artifact written");
18494        assert!(saved.contains("日本語のタイトル"));
18495        assert!(q.cwd.is_none());
18496        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18497        // A bad replacement is refused and asked about once more.
18498        let say = |id: &str, text: &str, choice: &str| {
18499            store
18500                .update(id, |q| {
18501                    q.thread.push(ask::Turn {
18502                        who: ask::Who::Operator,
18503                        body: text.to_owned(),
18504                        at: jiff::Timestamp::now(),
18505                        note: None,
18506                    });
18507                    q.answer(ask::Answer::Choice(choice.to_owned()))
18508                })
18509                .unwrap();
18510        };
18511        say(&q.id, "まだ日本語", "use my text");
18512        assert!(
18513            runner
18514                .guarded_pr_message(&winner, None, true)
18515                .await
18516                .unwrap()
18517                .is_none()
18518        );
18519        let second = find(&run);
18520        assert_ne!(second.id, q.id);
18521        assert!(second.detail.contains("did not pass"));
18522        // The retry still compares the original text; the rejected
18523        // replacement is named by category only.
18524        assert!(second.detail.contains("日本語のタイトル"));
18525        assert!(!second.detail.contains("まだ日本語"));
18526        assert!(second.detail.contains(&path.display().to_string()));
18527        say(&second.id, "fix: retry failed requests", "use my text");
18528        let message = runner
18529            .guarded_pr_message(&winner, None, true)
18530            .await
18531            .unwrap()
18532            .expect("a vetted replacement is posted");
18533        assert_eq!(message.title, "fix: retry failed requests");
18534        assert_eq!(
18535            runner
18536                .state
18537                .github_text
18538                .as_ref()
18539                .unwrap()
18540                .chosen_title
18541                .as_deref(),
18542            Some("fix: retry failed requests")
18543        );
18544        assert!(runner.state.github_text.as_ref().unwrap().resolved);
18545        // Resumed again: the saved decision is applied, nothing is re-asked.
18546        let again = runner
18547            .guarded_pr_message(&winner, None, true)
18548            .await
18549            .unwrap()
18550            .unwrap();
18551        assert_eq!(again.title, message.title);
18552        assert_eq!(
18553            store
18554                .list()
18555                .into_iter()
18556                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18557                .count(),
18558            2
18559        );
18560    }
18561
18562    #[test]
18563    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18564        // What magi itself writes stays English under any configured language,
18565        // so a future localisation of these headings fails here. (The agents'
18566        // own text is also checked by the posting gate.)
18567        let mut state = state_with_summary(
18568            "add retries",
18569            "TITLE: fix(web): batch reads\n- reads run.json once",
18570        );
18571        state.config.graph.language = "ja".to_owned();
18572        let m = pr_message(&state, 'A');
18573        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18574        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18575
18576        // The task is the operator's own text: it goes in untouched, and the
18577        // fallback title (no summary) may be in its language too.
18578        let task = "今回やってほしいこと: results projector を直す";
18579        let mut state = state_with_summary(task, "- no title line");
18580        state.config.graph.language = "ja".to_owned();
18581        let m = pr_message(&state, 'A');
18582        assert_eq!(
18583            m.title,
18584            format!("chore: land candidate A of run {}", state.id)
18585        );
18586        assert!(
18587            m.body.contains(&format!(
18588                "<summary>Original task</summary>\n\n{task}\n\n</details>"
18589            )),
18590            "{}",
18591            m.body
18592        );
18593    }
18594
18595    #[test]
18596    fn pr_message_scrubs_home_paths_and_addresses() {
18597        let state = state_with_summary(
18598            "fix it in /Users/someone/src/x",
18599            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18600        );
18601        let m = pr_message(&state, 'A');
18602        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18603            assert!(!m.body.contains(leak), "{}", m.body);
18604        }
18605        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18606    }
18607
18608    #[test]
18609    fn pr_message_survives_a_task_that_closes_details() {
18610        let state = state_with_summary("a </details> b", "TITLE: fix: x");
18611        let m = pr_message(&state, 'A');
18612        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18613    }
18614
18615    #[test]
18616    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18617        let cmd = manual_merge_command(
18618            MergeStyle::Squash,
18619            Path::new("/repo"),
18620            "b",
18621            "fix: \"quoted\" $(x) `y`\n\nbody",
18622        );
18623        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18624    }
18625
18626    #[test]
18627    fn manual_merge_command_matches_the_configured_style() {
18628        let repo = Path::new("/repo");
18629        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18630
18631        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18632        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18633
18634        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18635        assert_eq!(
18636            squash,
18637            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18638             \"Merge magi run 0832 (candidate A)\""
18639        );
18640
18641        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18642        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18643    }
18644
18645    #[test]
18646    fn a_nudge_gets_a_quarter_of_the_budget() {
18647        // The judge and implement budgets magi ships with.
18648        assert_eq!(retry_budget(secs(1200), true), secs(300));
18649        assert_eq!(retry_budget(secs(3600), true), secs(900));
18650    }
18651
18652    #[test]
18653    fn a_resent_prompt_keeps_the_whole_budget() {
18654        // The seat kept no context, so the retry is the original job again and
18655        // shortening it would only guarantee a second failure.
18656        assert_eq!(retry_budget(secs(1200), false), secs(1200));
18657        assert_eq!(retry_budget(secs(60), false), secs(60));
18658    }
18659
18660    #[test]
18661    fn the_floor_never_exceeds_the_original_budget() {
18662        // A short configured timeout must not be *raised* by the floor: the
18663        // operator asked for a bound, and a retry may not outlast the attempt
18664        // it is retrying.
18665        assert_eq!(retry_budget(secs(60), true), secs(60));
18666        assert_eq!(retry_budget(secs(480), true), secs(120));
18667        assert_eq!(retry_budget(secs(0), true), secs(0));
18668    }
18669
18670    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18671        agent::CommandEvidence {
18672            id: "item1".to_owned(),
18673            description: "cargo test".to_owned(),
18674            exit_code,
18675            result_summary: String::new(),
18676            source: "codex".to_owned(),
18677        }
18678    }
18679
18680    #[test]
18681    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18682        // No evidence is not the same fact as unconfirmed evidence: a
18683        // backend with no adapter, or a reply that ran no commands at all,
18684        // must not be misread as carrying a dangling job.
18685        assert!(!has_unconfirmed_command(&[]));
18686    }
18687
18688    #[test]
18689    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18690        // Deliberately not a check on the exit code's *value*: a fixer
18691        // legitimately runs something that fails mid-iteration before it
18692        // succeeds, and that must never by itself reopen a valid report.
18693        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18694        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18695        assert!(!has_unconfirmed_command(&[
18696            evidence(Some(0)),
18697            evidence(Some(101))
18698        ]));
18699    }
18700
18701    #[test]
18702    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18703        assert!(has_unconfirmed_command(&[
18704            evidence(Some(0)),
18705            evidence(None)
18706        ]));
18707    }
18708
18709    #[test]
18710    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18711        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18712        assert_eq!(
18713            verified_noop_claim(true, &[], text).as_deref(),
18714            Some("already fixed by b32cfc4, on main.")
18715        );
18716    }
18717
18718    #[test]
18719    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18720        // A timeout or a bad exit code reads as the ordinary loss it is,
18721        // whatever the reply's own prose claims.
18722        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18723        assert!(verified_noop_claim(false, &[], text).is_none());
18724    }
18725
18726    #[test]
18727    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18728        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18729        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18730        // A confirmed command alongside the marker is fine.
18731        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18732    }
18733
18734    #[test]
18735    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18736        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18737    }
18738
18739    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
18740    /// pair, in order, labelled A, B, C, ...
18741    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18742        runner.state.candidates = shape
18743            .iter()
18744            .enumerate()
18745            .map(|(i, &(empty, verified))| Candidate {
18746                index: i,
18747                label: (b'A' + i as u8) as char,
18748                agent: "sonnet".to_owned(),
18749                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18750                worktree: PathBuf::from(format!("/wt/{i}")),
18751                summary: String::new(),
18752                stat: String::new(),
18753                files: 0,
18754                commits: 0,
18755                empty,
18756                failed: None,
18757                verified_noop: verified.map(str::to_owned),
18758                duration_ms: 0,
18759                folded: false,
18760            })
18761            .collect();
18762    }
18763
18764    #[test]
18765    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18766        ask_test_home();
18767        let mut runner = runner_at(RunStatus::Implementing);
18768        set_candidates(
18769            &mut runner,
18770            &[
18771                (true, Some("already on main at b32cfc4")),
18772                (true, Some("same fix, see the existing test")),
18773            ],
18774        );
18775
18776        runner
18777            .after_implement()
18778            .expect("a verified no-op is not an error");
18779
18780        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18781    }
18782
18783    #[test]
18784    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18785        ask_test_home();
18786        let mut runner = runner_at(RunStatus::Implementing);
18787        // Candidate A declares a verified no-op; candidate B simply wrote
18788        // nothing and said nothing about why. One candidate's claim is not
18789        // the whole run's agreement.
18790        set_candidates(
18791            &mut runner,
18792            &[(true, Some("already on main at b32cfc4")), (true, None)],
18793        );
18794
18795        let err = runner
18796            .after_implement()
18797            .expect_err("an unverified empty candidate must still fail the run");
18798
18799        assert!(
18800            err.to_string().contains("no candidate produced a change"),
18801            "{err}"
18802        );
18803        assert_eq!(runner.state.status, RunStatus::Failed);
18804    }
18805
18806    #[test]
18807    fn after_implement_still_fails_an_ordinary_all_empty_run() {
18808        ask_test_home();
18809        let mut runner = runner_at(RunStatus::Implementing);
18810        set_candidates(&mut runner, &[(true, None), (true, None)]);
18811
18812        let err = runner
18813            .after_implement()
18814            .expect_err("no candidate declared anything; this is an ordinary failure");
18815
18816        assert!(
18817            err.to_string().contains("no candidate produced a change"),
18818            "{err}"
18819        );
18820        assert_eq!(runner.state.status, RunStatus::Failed);
18821    }
18822}
18823
18824/// A graft rollback whose restore could not be confirmed (see
18825/// [`Runner::graft`]).
18826#[derive(Debug)]
18827struct GraftRestoreFailed(String);
18828
18829impl std::fmt::Display for GraftRestoreFailed {
18830    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18831        write!(
18832            f,
18833            "could not restore the winner to its pre-graft commit: {}",
18834            self.0
18835        )
18836    }
18837}
18838
18839impl std::error::Error for GraftRestoreFailed {}
18840
18841/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
18842/// not the agent: they never draw on an agent's own timeout, and running out of
18843/// them never reads as an agent failure.
18844#[cfg(test)]
18845mod resource_wait_tests {
18846    use super::*;
18847    use std::collections::BTreeMap;
18848
18849    fn sh(id: &str, script: &str) -> AgentSpec {
18850        AgentSpec {
18851            id: id.to_owned(),
18852            kind: crate::config::AgentKind::Command,
18853            model: None,
18854            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18855            extra_args: Vec::new(),
18856            env: BTreeMap::new(),
18857            prompt_delivery: None,
18858        }
18859    }
18860
18861    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18862        SeatJob {
18863            seat: SeatState::new("impl-A", &spec.id, 1),
18864            spec,
18865            cwd: dir.to_path_buf(),
18866            prompt: "do it".to_owned(),
18867            timeout,
18868            allow_write: true,
18869            sessions: false,
18870            artifacts: dir.join("art"),
18871            stem: "impl-A".to_owned(),
18872            handover: None,
18873        }
18874    }
18875
18876    fn state(resource_wait_secs: u64) -> RunState {
18877        let mut config = Config::default();
18878        config.graph.timeout_resource_wait = resource_wait_secs;
18879        RunState::new(
18880            PathBuf::from("/repo"),
18881            "main".to_owned(),
18882            "abc1234".to_owned(),
18883            "task".to_owned(),
18884            config,
18885        )
18886    }
18887
18888    fn hold(cache: &Path) -> crate::cache::Guard {
18889        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18890        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18891            crate::cache::AcquireOutcome::Acquired(g) => g,
18892            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18893        }
18894    }
18895
18896    async fn run(
18897        jobs: Vec<SeatJob>,
18898        sem: Arc<Semaphore>,
18899        cache: Option<&Path>,
18900        st: &mut RunState,
18901    ) -> Vec<(usize, SeatState, AgentOutcome)> {
18902        let prompts = Prompts::default();
18903        let ctx = WaveCtx {
18904            run: "r",
18905            node: "implement",
18906            prompts: &prompts,
18907            cache,
18908            round: None,
18909            carry_seats: false,
18910            pauses: None,
18911        };
18912        wave(jobs, sem, &ctx, st, 0).await
18913    }
18914
18915    #[tokio::test]
18916    async fn verification_cache_wait_preserves_the_command_budget() {
18917        crate::run::pin_test_home();
18918        let tmp = tempfile::tempdir().unwrap();
18919        let cache = tmp.path().join("target");
18920        let guard = hold(&cache);
18921        let release = tokio::spawn(async move {
18922            tokio::time::sleep(Duration::from_millis(100)).await;
18923            guard.release();
18924        });
18925        let mut st = state(10);
18926        let budget = Duration::from_millis(50);
18927        let wait = repository_wait(&st, None);
18928        let (outcomes, _) = with_cache_lease(
18929            &mut st,
18930            Some(&cache),
18931            "gate",
18932            "gate",
18933            tmp.path(),
18934            "abc1234",
18935            budget,
18936            "test gate",
18937            &wait,
18938            |_, available| async move {
18939                assert_eq!(
18940                    available, budget,
18941                    "contention must not shorten verification"
18942                );
18943                (Vec::new(), false, Vec::new())
18944            },
18945        )
18946        .await
18947        .unwrap();
18948        release.await.unwrap();
18949        assert!(outcomes.is_empty(), "{outcomes:?}");
18950        assert!(
18951            st.events
18952                .iter()
18953                .any(|event| event.message.contains("waiting for"))
18954        );
18955        assert!(st.resource_wait.is_none());
18956        // The verification body no longer owns the lease after returning.
18957        hold(&cache).release();
18958    }
18959
18960    #[tokio::test]
18961    async fn verification_cache_wait_expires_without_starting_commands() {
18962        crate::run::pin_test_home();
18963        let tmp = tempfile::tempdir().unwrap();
18964        let cache = tmp.path().join("target");
18965        let _held = hold(&cache);
18966        let mut st = state(0);
18967        let wait = repository_wait(&st, None);
18968        let (outcomes, _) = with_cache_lease(
18969            &mut st,
18970            Some(&cache),
18971            "gate",
18972            "gate",
18973            tmp.path(),
18974            "abc1234",
18975            Duration::from_secs(60),
18976            "test gate",
18977            &wait,
18978            |_, _| async { panic!("commands must not start without the cache lease") },
18979        )
18980        .await
18981        .unwrap();
18982        assert_eq!(outcomes.len(), 1);
18983        assert!(outcomes[0].resource_blocked);
18984        assert!(st.resource_wait.is_none());
18985    }
18986
18987    #[tokio::test]
18988    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18989        crate::run::pin_test_home();
18990        let tmp = tempfile::tempdir().unwrap();
18991        let cache = tmp.path().join("target-a");
18992        let guard = hold(&cache);
18993        // Held for longer than the agent's whole 1s budget.
18994        tokio::spawn(async move {
18995            tokio::time::sleep(Duration::from_millis(1500)).await;
18996            guard.release();
18997        });
18998        let mut st = state(60);
18999        let results = run(
19000            vec![job(
19001                sh("alpha", "echo ok"),
19002                tmp.path(),
19003                Duration::from_secs(1),
19004            )],
19005            Arc::new(Semaphore::new(1)),
19006            Some(&cache),
19007            &mut st,
19008        )
19009        .await;
19010        assert!(
19011            matches!(results[0].2, AgentOutcome::Ok(_)),
19012            "the wait must not be charged to the agent"
19013        );
19014        assert!(st.resource_block.is_none());
19015        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19016        assert!(st.resource_wait.is_none() && st.active.is_empty());
19017    }
19018
19019    #[tokio::test]
19020    #[cfg(unix)]
19021    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19022        use std::os::fd::AsRawFd as _;
19023        crate::run::pin_test_home();
19024        let tmp = tempfile::tempdir().unwrap();
19025        crate::git::git(tmp.path(), &["init"]).await.unwrap();
19026        let common = tmp.path().join(".git");
19027        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19028        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19029        assert_eq!(
19030            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19031            0
19032        );
19033        let mut st = state(1);
19034        st.config.coordination = crate::config::test_coordination();
19035        let mut reader = job(
19036            sh("reader", "echo must-not-run"),
19037            tmp.path(),
19038            Duration::from_secs(30),
19039        );
19040        reader.allow_write = false;
19041        let results = run(
19042            vec![reader.clone()],
19043            Arc::new(Semaphore::new(1)),
19044            None,
19045            &mut st,
19046        )
19047        .await;
19048        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19049        assert_eq!(results[0].1.turns, 0);
19050        assert!(FailClass::of(&results[0].2).is_none());
19051        assert_eq!(
19052            st.resource_block.as_ref().unwrap().kind,
19053            "repository coordination"
19054        );
19055        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19056        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19057        drop(external);
19058        let editor = crate::repo_lock::SeatLease::for_checkout(
19059            tmp.path(),
19060            true,
19061            crate::config::test_coordination().as_ref(),
19062        )
19063        .await
19064        .unwrap();
19065        let pause = Pause::new();
19066        let interrupt = Pause::new();
19067        let prompts = Prompts::default();
19068        let ctx = WaveCtx {
19069            run: "r",
19070            node: "review",
19071            prompts: &prompts,
19072            cache: None,
19073            round: None,
19074            carry_seats: false,
19075            pauses: Some((&pause, &interrupt)),
19076        };
19077        let (results, ()) = tokio::join!(
19078            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19079            async {
19080                tokio::time::sleep(Duration::from_millis(50)).await;
19081                pause.park();
19082            }
19083        );
19084        assert!(
19085            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19086        );
19087        assert_eq!(results[0].1.turns, 0);
19088        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19089        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19090        drop(editor);
19091    }
19092
19093    #[tokio::test]
19094    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19095        crate::run::pin_test_home();
19096        let tmp = tempfile::tempdir().unwrap();
19097        let cache = tmp.path().join("target-b");
19098        let _held = hold(&cache);
19099        let mut st = state(1);
19100        let results = run(
19101            vec![job(
19102                sh("alpha", "echo ok"),
19103                tmp.path(),
19104                Duration::from_secs(1),
19105            )],
19106            Arc::new(Semaphore::new(1)),
19107            Some(&cache),
19108            &mut st,
19109        )
19110        .await;
19111        let (_, seat, out) = &results[0];
19112        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19113        assert!(wave_blocked(&results));
19114        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19115        assert_eq!(seat.turns, 0, "nothing was asked");
19116        assert!(
19117            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19118            "no agent was spawned"
19119        );
19120        assert!(FailClass::of(out).is_none(), "not a failure class");
19121        let block = st.resource_block.as_ref().expect("recorded for the resume");
19122        assert_eq!(block.kind, "shared build cache");
19123        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19124        assert!(st.active.is_empty());
19125    }
19126
19127    #[tokio::test]
19128    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19129        crate::run::pin_test_home();
19130        let tmp = tempfile::tempdir().unwrap();
19131        let mut st = state(1);
19132        let results = run(
19133            vec![job(
19134                sh("alpha", "echo ok"),
19135                tmp.path(),
19136                Duration::from_secs(1),
19137            )],
19138            Arc::new(Semaphore::new(0)),
19139            None,
19140            &mut st,
19141        )
19142        .await;
19143        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19144        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19145        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19146    }
19147
19148    #[tokio::test]
19149    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19150        crate::run::pin_test_home();
19151        let tmp = tempfile::tempdir().unwrap();
19152        let mut st = state(60);
19153        let results = run(
19154            vec![job(
19155                sh("alpha", "sleep 30"),
19156                tmp.path(),
19157                Duration::from_secs(1),
19158            )],
19159            Arc::new(Semaphore::new(1)),
19160            None,
19161            &mut st,
19162        )
19163        .await;
19164        let out = &results[0].2;
19165        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19166        let class = FailClass::of(out).expect("a failure");
19167        assert_eq!(class, FailClass::Timeout);
19168        assert!(should_hand_over(None, &class), "existing handover policy");
19169        assert!(st.resource_block.is_none());
19170    }
19171}