Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's Git administration access, network and repository coordination \
526         before retrying; an unattended seat cannot approve those permissions"
527    )
528}
529
530/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
531/// on an early return or a panic.
532///
533/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
534/// manual `magi fix` invocations against the same run are otherwise
535/// invisible to each other and would race to remove and recreate the same
536/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
537/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
538/// which is only ever reclaimed later, out of band, by
539/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
540/// `magi fix` invocation is not necessarily running under either of those, so
541/// nothing would ever sweep a lock a killed or crashed process left behind.
542/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
543/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
544/// lease use: an unreadable or unparsable pid, or a liveness query the
545/// platform cannot answer, reads as alive and the lock is left in place.
546struct FixClaim {
547    path: PathBuf,
548}
549
550impl FixClaim {
551    fn acquire(dir: &Path) -> Result<Self> {
552        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553        let path = dir.join("fix.lock");
554        let retry = crate::lock_file::Retry::new(None);
555        match Self::create(&path, retry) {
556            Ok(claim) => Ok(claim),
557            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558                if Self::reclaim_if_dead(&path) {
559                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560                } else {
561                    bail!(
562                        "another repair or `magi fix` is already running for this run ({} exists)",
563                        path.display()
564                    )
565                }
566            }
567            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568        }
569    }
570
571    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572        let mut f = retry.create(path)?;
573        use std::io::Write as _;
574        // Read back by `reclaim_if_dead` on a later, stuck invocation.
575        writeln!(f, "{}", std::process::id())?;
576        Ok(Self {
577            path: path.to_owned(),
578        })
579    }
580
581    /// True if the lock named a process confirmed dead, in which case it was
582    /// also removed. Never true on an unreadable file, an unparsable pid, or
583    /// a liveness query the platform cannot answer — see this type's own doc.
584    fn reclaim_if_dead(path: &Path) -> bool {
585        let dead = std::fs::read_to_string(path)
586            .ok()
587            .and_then(|body| body.trim().parse::<u32>().ok())
588            .is_some_and(|pid| !crate::proc::pid_alive(pid));
589        dead && std::fs::remove_file(path).is_ok()
590    }
591}
592
593impl Drop for FixClaim {
594    fn drop(&mut self) {
595        let _ = std::fs::remove_file(&self.path);
596    }
597}
598
599/// Holds the mutable runner as well as its durable lease: cancellation must
600/// clear both copies of ownership, including state a caller may save later.
601struct DriverExecution<'a> {
602    runner: &'a mut Runner,
603    driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607    fn drop(&mut self) {
608        self.driver.release_local(&mut self.runner.state);
609    }
610}
611
612impl Runner {
613    /// Start a fresh run against `repo`.
614    pub async fn start(
615        repo: &Path,
616        instruction: String,
617        config: Config,
618        origin: Origin,
619    ) -> Result<Self> {
620        Self::start_naming(repo, instruction, "", config, origin).await
621    }
622
623    /// [`Runner::start`] for a queued task. Titles and instructions never
624    /// select implementation sources. The title argument is retained for callers.
625    pub async fn start_naming(
626        repo: &Path,
627        instruction: String,
628        _also_scan: &str,
629        config: Config,
630        origin: Origin,
631    ) -> Result<Self> {
632        let repo = git::toplevel(repo).await?;
633        let missing = agent::missing_programs(&config.agents);
634        if !missing.is_empty() {
635            bail!(
636                "these agent programs are not on PATH: {}. Fix the roster in \
637                 magi.toml or install them.",
638                missing.join(", ")
639            );
640        }
641        let base_branch =
642            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643                .await?;
644        let fetched_base = resolve_base(
645            &repo,
646            &base_branch,
647            &config.merge.remote,
648            config.coordination.as_ref(),
649        )
650        .await?;
651        let base_commit = fetched_base.commit.clone();
652        let roles = config.resolve_roles()?;
653        let max_parallel = config.graph.max_parallel.max(1);
654        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655        // Recorded before the first save, so a crash right after minting
656        // cannot leave a run with no origin. Legibility only: nothing reads it
657        // to decide anything.
658        state.fetched_base = Some(fetched_base);
659        state.origin = Some(origin);
660        state.event("start", format!("run {} created", state.id));
661        state.save()?;
662        Ok(Self {
663            state,
664            roles,
665            sem: Arc::new(Semaphore::new(max_parallel)),
666            pause: Pause::new(),
667            interrupt: Pause::new(),
668        })
669    }
670
671    /// Open a review-only run against work that already exists on `branch`.
672    ///
673    /// The expensive half of the graph is the implement wave — measured at
674    /// 111 and 134 internal tool-loop turns on this repository, against a
675    /// handful for a judge or a reviewer. The cheap half is worth running on
676    /// hand-written work too, and there was no way to reach it.
677    ///
678    /// No new state and no schema change are needed: a run with **one** viable
679    /// candidate and a tally already decided degrades `execute` to exactly
680    /// review → gate → merge, because `judge` skips a single-candidate field,
681    /// `deliberate` has fewer than two first choices to reconcile, `vote`
682    /// returns early, `tally` is already present and `fold_losers` has no
683    /// losers. Resuming such a run therefore does the right thing as well.
684    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685        Self::review_taking_over(repo, branch, config, None, origin).await
686    }
687
688    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
689    /// at the same task still has `branch` checked out, its worktree is
690    /// released first if that is safe (see [`crate::handover`]), and the
691    /// review refuses with the reason if it is not. `None` is a hand-run
692    /// review: it has no earlier attempts, so only a worktree of a dead run
693    /// magi recorded itself can be released.
694    pub async fn review_taking_over(
695        repo: &Path,
696        branch: &str,
697        config: Config,
698        takeover: Option<crate::handover::Takeover>,
699        origin: Origin,
700    ) -> Result<Self> {
701        let repo = git::toplevel(repo).await?;
702        let missing = agent::missing_programs(&config.agents);
703        if !missing.is_empty() {
704            bail!(
705                "these agent programs are not on PATH: {}. Fix the roster in \
706                 magi.toml or install them.",
707                missing.join(", ")
708            );
709        }
710        let base_branch =
711            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712                .await?;
713        if base_branch == branch {
714            bail!("`{branch}` is the base branch; there is nothing to review against");
715        }
716        let fetched_base = resolve_base(
717            &repo,
718            &base_branch,
719            &config.merge.remote,
720            config.coordination.as_ref(),
721        )
722        .await?;
723        let base_commit = fetched_base.commit.clone();
724
725        let roles = config.resolve_roles()?;
726        let max_parallel = config.graph.max_parallel.max(1);
727        let mut state = RunState::new(
728            repo.clone(),
729            base_branch,
730            base_commit.clone(),
731            String::new(),
732            config,
733        );
734        state.fetched_base = Some(fetched_base);
735        state.origin = Some(origin);
736
737        // Released before anything else touches the branch: a stale local
738        // branch is moved with `git branch -f`, which git refuses while an
739        // earlier attempt's worktree still has it checked out. Everything
740        // after this point that can fail puts the old run back.
741        // A hand-run review has no task, hence no earlier attempts, but a
742        // worktree of a dead run magi made may still be released.
743        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744            earlier: Vec::new(),
745            home: crate::run::home(),
746            choice: None,
747            task_instruction: None,
748        });
749        let wait = repository_wait(&state, None);
750        let coordination_config = state.config.coordination.clone();
751        let coordination =
752            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753                .await?;
754        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755        if let Some(released) = &released {
756            state.event(
757                "release",
758                format!(
759                    "took `{branch}` over from run {}: its worktree was released: {}",
760                    crate::run::short_of(&released.old_id),
761                    released.audit
762                ),
763            );
764        }
765        // The owner's answer to an earlier divergence question is applied
766        // here: after the release (git will not move a checked-out branch)
767        // and before the sync that would otherwise ask again.
768        if let Some(choice) = takeover.choice.as_ref()
769            && let Err(e) =
770                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771                    .await
772        {
773            if let Some(released) = &released {
774                released.restore(&repo, branch).await;
775            }
776            return Err(e.context("applying the owner's answer about the diverged branch"));
777        }
778        // An earlier attempt of this task that published the task head, and
779        // whose work is the branch under review, hands that head over: the
780        // review run updates the same pull request. Without one (a legacy
781        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
782        // own head and its own open pull request is adopted as before.
783        for id in takeover.earlier.iter().rev() {
784            if let Ok(old) = RunState::load_under(id, &takeover.home)
785                && let Some(head) = old.pr_head.clone()
786                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787            {
788                state.pr_head = Some(head);
789                break;
790            }
791        }
792        drop(coordination);
793        let opened = Self::open_review(
794            &repo,
795            branch,
796            state,
797            roles,
798            max_parallel,
799            base_commit,
800            &takeover,
801        )
802        .await;
803        if opened.is_err()
804            && let Some(released) = &released
805        {
806            let _coordination = crate::repo_lock::SeatLease::operation_with(
807                &repo,
808                coordination_config.as_ref(),
809                &wait,
810            )
811            .await?;
812            released.restore(&repo, branch).await;
813        }
814        opened
815    }
816
817    /// Recover exactly the operator-selected public implementation, without touching old sources.
818    pub async fn recover_pr(
819        repo: &Path,
820        task: &str,
821        selection: crate::pr_recovery::Selection,
822        mut config: Config,
823    ) -> Result<Self> {
824        selection.configure(&mut config)?;
825        selection.check_task(task, &selection.instruction)?;
826        let base_branch =
827            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828                .await?;
829        let fetched_base = resolve_base(
830            repo,
831            &base_branch,
832            &config.merge.remote,
833            config.coordination.as_ref(),
834        )
835        .await?;
836        let base_commit = fetched_base.commit.clone();
837        selection
838            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839            .await?;
840        Box::pin(selection.verify_source(repo)).await?;
841        if let Some(proof) = &selection.source_proof {
842            anyhow::ensure!(
843                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844                "source rebase base is outside the fetched landing base"
845            );
846        }
847        let roles = config.resolve_roles()?;
848        let max_parallel = config.graph.max_parallel.max(1);
849        let mut state = RunState::new(
850            repo.to_path_buf(),
851            base_branch,
852            base_commit.clone(),
853            selection.instruction.clone(),
854            config,
855        );
856        state.fetched_base = Some(fetched_base);
857        state.origin = Some(crate::run::Origin::queue(task));
858        state.pr_head = Some(selection.branch.clone());
859        state.pr_recovery = Some(selection.clone());
860        crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862    }
863
864    async fn open_selected(
865        repo: &Path,
866        state: RunState,
867        roles: ResolvedRoles,
868        max_parallel: usize,
869        base_commit: String,
870        selection: crate::pr_recovery::Selection,
871    ) -> Result<Self> {
872        let mut state = state;
873        let branch = state.branch_for('A');
874        let wait = repository_wait(&state, None);
875        let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876        git::git(repo, &["branch", &branch, selection.source()]).await?;
877        drop(coordination);
878        state.event(
879            "recovery",
880            format!(
881                "selected implementation {} for {} (public lease {})",
882                selection.source(),
883                selection.url,
884                selection.head
885            ),
886        );
887        let takeover = crate::handover::Takeover {
888            earlier: Vec::new(),
889            home: crate::run::home(),
890            choice: None,
891            task_instruction: Some(selection.instruction.clone()),
892        };
893        state.save()?;
894        let id = state.id.clone();
895        let opened = Self::open_review(
896            repo,
897            &branch,
898            state,
899            roles,
900            max_parallel,
901            base_commit,
902            &takeover,
903        )
904        .await;
905        match opened {
906            Ok(runner) => Ok(runner),
907            Err(e) => {
908                if let Ok(mut partial) = RunState::load(&id) {
909                    partial.status = RunStatus::Blocked;
910                    partial.driver_exited = true;
911                    partial.event(
912                        "recovery",
913                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
914                    );
915                    partial.save()?;
916                }
917                Err(crate::pr_recovery::StartupFailed {
918                    run: id,
919                    cause: format!("{e:#}"),
920                }
921                .into())
922            }
923        }
924    }
925
926    /// What the newest earlier attempt that reviewed `branch` leaves for this
927    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
928    /// no earlier attempts. Reads the earlier record only: ownership of its
929    /// worktree and branch is [`crate::handover`]'s business.
930    async fn inherit(
931        repo: &Path,
932        branch: &str,
933        base_commit: &str,
934        takeover: &crate::handover::Takeover,
935    ) -> Option<crate::recovery::RecoveryRecord> {
936        use crate::recovery::{Relation, assess};
937        let newest = takeover.earlier.last()?;
938        let mut chosen = None;
939        for id in takeover.earlier.iter().rev() {
940            if let Ok(old) = RunState::load_under(id, &takeover.home)
941                && !old.reviews.is_empty()
942                && (old.winner().is_some_and(|w| w.branch == branch)
943                    || old.pr_head.as_deref() == Some(branch))
944            {
945                chosen = Some(old);
946                break;
947            }
948        }
949        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950            .await
951            .unwrap_or_default();
952        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953            Some(last) if last.head == tip => Relation::Same,
954            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955            _ => Relation::Diverged,
956        };
957        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958        Some(assess(from, chosen.as_ref(), base_commit, relation))
959    }
960
961    /// The half of [`Runner::review_taking_over`] that can fail after an
962    /// earlier attempt's worktree was released.
963    async fn open_review(
964        repo: &Path,
965        branch: &str,
966        mut state: RunState,
967        roles: ResolvedRoles,
968        max_parallel: usize,
969        base_commit: String,
970        takeover: &crate::handover::Takeover,
971    ) -> Result<Self> {
972        let wait = repository_wait(&state, None);
973        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974        if state.pr_recovery.is_none() {
975            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976        }
977        // The commit subjects are the closest thing to a task statement that
978        // existing work carries, and the reviewers are told as much.
979        let start = review_base(
980            repo,
981            &state.config.merge.remote,
982            &state.base_branch,
983            &base_commit,
984            branch,
985        )
986        .await;
987        let log = git::log_oneline(repo, &start, branch)
988            .await
989            .unwrap_or_default();
990        let log_text = if log.trim().is_empty() {
991            "(no commit messages)"
992        } else {
993            log.trim()
994        };
995        let mut instruction = format!(
996            "Review the work already on branch `{branch}`. There is no task \
997             statement: what the change claims to do is whatever its commits \
998             say.\n\n{log_text}"
999        );
1000        // A retry of a task whose earlier attempt left reviews behind carries
1001        // that attempt's evidence, and the task's own statement with it: the
1002        // completion criteria are the task's, not whatever the commits claim.
1003        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004            if let Some(task) = takeover
1005                .task_instruction
1006                .as_deref()
1007                .filter(|t| !t.trim().is_empty())
1008                && rec.reviewed_head.is_some()
1009            {
1010                instruction = format!(
1011                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012                    task.trim()
1013                );
1014                rec.criteria_inherited = true;
1015            }
1016            state.event("recovery", rec.summary());
1017            state.recovery = Some(rec);
1018        }
1019        if let Some(selected) = &state.pr_recovery {
1020            instruction = selected.instruction.clone();
1021        }
1022        state.task_instruction = takeover.task_instruction.clone();
1023        state.instruction = instruction;
1024        state.reviewed_commits = Some(
1025            git::subjects(repo, &start, branch)
1026                .await
1027                .unwrap_or_default(),
1028        );
1029
1030        // A takeover must not forget a hold that a previous review observed,
1031        // even if repairs have since removed it from the branch.
1032        for id in &takeover.earlier {
1033            if let Ok(old) = RunState::load_under(id, &takeover.home)
1034                && old.repo == state.repo
1035                && (old.pr_head.as_deref() == Some(branch)
1036                    || old.winner().is_some_and(|w| w.branch == branch))
1037            {
1038                for hold in old.branch_owner_holds {
1039                    if !state.branch_owner_holds.contains(&hold) {
1040                        state.event(
1041                            "approval-policy",
1042                            format!(
1043                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1044                                hold.control.key(),
1045                                hold.file,
1046                                hold.head
1047                            ),
1048                        );
1049                        state.branch_owner_holds.push(hold);
1050                    }
1051                }
1052            }
1053        }
1054        let reviewed_head = git::rev_parse(repo, branch).await?;
1055        state.observe_branch_owner_holds(&reviewed_head).await;
1056
1057        // An attached worktree, so the fixer's commits land on the branch under
1058        // review rather than on a detached head nobody will look at again.
1059        let worktree = state.worktree_root().join("under-review");
1060        if let Some(parent) = worktree.parent() {
1061            tokio::fs::create_dir_all(parent).await.ok();
1062        }
1063        let path = worktree.to_string_lossy().to_string();
1064        git::git(repo, &["worktree", "add", &path, branch])
1065            .await
1066            .with_context(|| {
1067                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1068            })?;
1069
1070        let setup = match ensure_setup_config(&mut state, repo).await {
1071            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1072            Err(e) => Err(e),
1073        };
1074        if let Err(e) = setup {
1075            git::worktree_remove_clean(repo, &worktree).await.ok();
1076            return Err(e);
1077        }
1078        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1079            .await
1080            .unwrap_or(0);
1081        if commits == 0 {
1082            git::worktree_remove_clean(repo, &worktree).await.ok();
1083            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1084        }
1085        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1086            .await
1087            .map(|f| f.len())
1088            .unwrap_or(0);
1089        if files == 0
1090            && let (Ok(head_tree), Ok(base_tree)) = (
1091                git::tree_of(&worktree, "HEAD").await,
1092                git::tree_of(&worktree, &base_commit).await,
1093            )
1094            && head_tree == base_tree
1095        {
1096            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1097            git::worktree_remove_clean(repo, &worktree).await.ok();
1098            bail!(
1099                "`{branch}` at {} has a tree identical to base {}; this usually means \
1100                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1101                 against `{}/{branch}`) rather than an empty change",
1102                short(&head),
1103                short(&base_commit),
1104                state.config.merge.remote
1105            );
1106        }
1107        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1108            .await
1109            .unwrap_or_default();
1110
1111        state.candidates.push(Candidate {
1112            index: 0,
1113            label: 'A',
1114            // Not an agent id on purpose: nothing in the roster wrote this, and
1115            // the stats tables must not credit anyone with a win for it.
1116            agent: EXISTING_BRANCH.to_owned(),
1117            branch: branch.to_owned(),
1118            worktree,
1119            summary: String::new(),
1120            stat,
1121            files,
1122            commits,
1123            empty: false,
1124            failed: None,
1125            verified_noop: None,
1126            duration_ms: 0,
1127            folded: false,
1128        });
1129        state.tally = Some(Tally {
1130            first_choice: BTreeMap::from([('A', 0)]),
1131            borda: BTreeMap::new(),
1132            winner: 'A',
1133            rankings: 0,
1134            unanimous_initial: false,
1135            deliberated: false,
1136            changed_votes: 0,
1137            unanimous_final: false,
1138            tie_break: None,
1139            // No panel sat, so no quorum applies. Zero judges is the correct
1140            // number for work that never competed, and must not be reported as
1141            // a collapsed panel.
1142            judges: 0,
1143            present: 0,
1144            quorum: 0,
1145            met_quorum: true,
1146            uncontested: Some("review-only run: nothing competed".to_owned()),
1147        });
1148        state.status = RunStatus::Reviewing;
1149        state.prep_complete = Some(true);
1150        state.event(
1151            "start",
1152            format!(
1153                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1154                state.id
1155            ),
1156        );
1157        state.save()?;
1158        Ok(Self {
1159            state,
1160            roles,
1161            sem: Arc::new(Semaphore::new(max_parallel)),
1162            pause: Pause::new(),
1163            interrupt: Pause::new(),
1164        })
1165    }
1166
1167    /// Reopen an existing run.
1168    pub fn resume(id: &str) -> Result<Self> {
1169        let state = RunState::load(id)?;
1170        crate::pr_head::ensure_owner(&state)?;
1171        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1172            bail!(
1173                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1174            );
1175        }
1176        if let Some(to) = &state.released_to {
1177            bail!(
1178                "run {} cannot be resumed: its worktree was released to run {}",
1179                state.short(),
1180                crate::run::short_of(to)
1181            );
1182        }
1183        let roles = state.config.resolve_roles()?;
1184        if let Some(id) = &state.config.graph.selected_implementer
1185            && state.candidates.iter().any(|c| c.agent != *id)
1186        {
1187            bail!(
1188                "replacement implementer pin does not match the saved candidates; task must stay held"
1189            );
1190        }
1191        let max_parallel = state.config.graph.max_parallel.max(1);
1192        Ok(Self {
1193            state,
1194            roles,
1195            sem: Arc::new(Semaphore::new(max_parallel)),
1196            pause: Pause::new(),
1197            interrupt: Pause::new(),
1198        })
1199    }
1200
1201    /// Walk the graph to a terminal state, skipping nodes already recorded.
1202    ///
1203    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1204    /// the phone - ends here, so this is the one place a run that ended
1205    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1206    /// notification centre. Best-effort: see [`crate::notices::raise`].
1207    pub async fn execute(&mut self) -> Result<()> {
1208        if let Some(selection) = &self.state.pr_recovery
1209            && (selection.source_proof.is_some() || selection.source_run.is_some())
1210        {
1211            Box::pin(selection.verify_source(&self.state.repo)).await?;
1212        }
1213        // Queue claims cover daemon work; this also serializes manual drivers
1214        // consuming a repair answer, including the new review and decision.
1215        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1216            || self.state.land_approval.is_some()
1217            || self.state.merge_decision.is_some()
1218            || !self.state.merge_repairs.is_empty()
1219            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1220        {
1221            let claim = FixClaim::acquire(&self.state.dir())?;
1222            let disk = RunState::load(&self.state.id)?;
1223            let checkpoint = |s: &RunState| {
1224                (
1225                    s.merge_repairs.len(),
1226                    s.merge_repairs
1227                        .last()
1228                        .map(|r| (r.question.clone(), r.phase)),
1229                    s.land_approval
1230                        .as_ref()
1231                        .map(|a| (a.question.clone(), a.head.clone())),
1232                )
1233            };
1234            if checkpoint(&disk) != checkpoint(&self.state) {
1235                bail!(
1236                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1237                );
1238            }
1239            Some(claim)
1240        } else {
1241            None
1242        };
1243        let mut execution = self.claim_driver()?;
1244        let runner = &mut *execution.runner;
1245        let wait = runner.repository_wait();
1246        // Keep the graph's large future off the callers' stack, including
1247        // standalone task attempts running on a default-sized test thread.
1248        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1249        let result = match result {
1250            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1251                record_repository_block(&mut runner.state, &e);
1252                runner.park_here()?;
1253                Ok(())
1254            }
1255            other => other,
1256        };
1257        execution.driver.finish(&mut runner.state)?;
1258        let ended = if result.is_err() {
1259            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1260        } else {
1261            crate::notices::run_ended(&runner.state)
1262        };
1263        if let Some(notice) = ended {
1264            crate::notices::raise(notice);
1265        }
1266        result
1267    }
1268
1269    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1270        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1271    }
1272
1273    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1274        let wait = self.repository_wait();
1275        coordinate_operation(&mut self.state, cwd, &wait).await
1276    }
1277
1278    /// Validate ownership and persist the driver claim in one critical section.
1279    fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1280        let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1281            Ok(owner) => owner,
1282            Err(error) => {
1283                let detail = format!("worktree maintenance or ownership update: {error:#}");
1284                // Admission did not own run.json: retain this resource outcome
1285                // in memory for queue settlement without overwriting a concurrent
1286                // retirement or publication transaction.
1287                self.state.resource_block = Some(crate::run::ResourceBlock {
1288                    node: "start".into(),
1289                    kind: "worktree maintenance".into(),
1290                    detail: detail.clone(),
1291                    at: Timestamp::now(),
1292                });
1293                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1294            }
1295        };
1296        let disk = RunState::load(&self.state.id)?;
1297        crate::pr_head::ensure_owner(&disk)?;
1298        if disk.liveness(false) == crate::run::Liveness::Live {
1299            bail!("run already has a live driver; wait for its execution to end");
1300        }
1301        if let Some(to) = &disk.released_to {
1302            bail!(
1303                "run {} cannot continue: its worktree was released to run {}",
1304                self.state.short(),
1305                crate::run::short_of(to)
1306            );
1307        }
1308        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1309        let mut execution = DriverExecution {
1310            runner: self,
1311            driver,
1312        };
1313        // Declare after the execution guard: on any claim error this short
1314        // lock must drop before that guard reloads and releases ownership.
1315        let _owner = owner;
1316        let state = &mut execution.runner.state;
1317        // Moving again, so it is no longer parked. Set before the walk rather
1318        // than in `resume`, so every way of re-entering the graph clears it
1319        // and a card cannot claim a run is waiting to be resumed while the
1320        // agents are already working.
1321        state.parked = false;
1322        state.resource_wait = None;
1323        state.resource_block = None;
1324        // Any seat this state still lists as answering belongs to whatever
1325        // process last drove this run — this one included, if it crashed
1326        // mid-wave. Cleared and flushed immediately, before anything else
1327        // runs, so a resume can never show a seat as live when nothing is
1328        // asking it anything yet; the node that actually dispatches the next
1329        // wave repopulates it.
1330        state.clear_active();
1331        // Recorded in the same spot, and flushed together with the clear
1332        // above: this is the pid a reader checks (`RunState::liveness`) when
1333        // no daemon claim exists to answer "is a process still driving this
1334        // run" — a plain `magi run` / `magi review` typed into a terminal
1335        // claims nothing there. Always overwritten, never only-if-absent, so
1336        // a resumed run's stale pid from a previous, possibly-dead process
1337        // can never survive into this one's own report. Unlike
1338        // `clear_active`, this changes on every single `execute()` call, so
1339        // the save below is now unconditional rather than only-if-cleared.
1340        //
1341        // `driver_started_at` is recorded in the same breath, from this same
1342        // pid, so `liveness` can tell a live pid that is genuinely still us
1343        // apart from one the OS has since handed to an unrelated process —
1344        // see that field's own doc for why the pid alone is not enough.
1345        let pid = std::process::id();
1346        state.driver_pid = Some(pid);
1347        state.driver_started_at = crate::proc::process_started_at(pid);
1348        state.driver_exited = false;
1349        state.begin_pass();
1350        execution.driver.bind(state);
1351        state.save()?;
1352        execution.driver.record(state)?;
1353        Ok(execution)
1354    }
1355
1356    async fn execute_graph(&mut self) -> Result<()> {
1357        self.state.configure_recovery_resume()?;
1358        // Preservation may invalidate an old empty classification. Resolve it
1359        // before any resume shortcut can reuse a verdict or publish a branch.
1360        self.persist_candidates().await?;
1361        if self
1362            .state
1363            .merge_decision
1364            .as_ref()
1365            .is_some_and(|decision| !decision.closed)
1366            && self.state.status != RunStatus::Landing
1367        {
1368            if crate::merge_decision::resumable_hold(&self.state) {
1369                self.state.merge = Some(MergeOutcome {
1370                    mode: MergeMode::Pr,
1371                    ok: false,
1372                    detail: self
1373                        .state
1374                        .merge_decision
1375                        .as_ref()
1376                        .unwrap()
1377                        .target
1378                        .pr
1379                        .clone(),
1380                    empty: false,
1381                });
1382                self.run_land().await?;
1383                self.settle_questions();
1384                return Ok(());
1385            }
1386            bail!(
1387                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1388            );
1389        }
1390        if self.state.config.graph.selected_implementer.is_some()
1391            && self.state.status == RunStatus::Blocked
1392            && self.state.tally.is_none()
1393            && self.state.candidate_preservation.is_empty()
1394            && self.state.interrupted_implementations.is_empty()
1395        {
1396            bail!(
1397                "selected implementation is held; explicitly release the owning task to retry it"
1398            );
1399        }
1400
1401        if self.state.merge_repairs.last().is_some_and(|r| {
1402            matches!(
1403                r.phase,
1404                crate::run::MergeRepairPhase::Requested
1405                    | crate::run::MergeRepairPhase::Fixing
1406                    | crate::run::MergeRepairPhase::Reviewing
1407            )
1408        }) {
1409            if self.repair_merge_review().await? {
1410                self.run_land().await?;
1411            }
1412            self.settle_questions();
1413            return Ok(());
1414        }
1415        // Publication already passed review and gate. Resume only that node:
1416        // syncing to a newer base here could change the verified commit.
1417        if self.state.pr_publication_pending {
1418            self.merge().await?;
1419            if self.resource_halt()? {
1420                return Ok(());
1421            }
1422            self.state.save()?;
1423            self.settle_questions();
1424            return Ok(());
1425        }
1426        // A run that already lost its quorum never resumes into the verdict
1427        // machinery: `deliberate` and `vote` would otherwise clobber the
1428        // stalled marker back to Voting and the run would keep going past a
1429        // verdict that is no longer trustworthy. Everything already recorded is
1430        // kept, so the run stays resumable (or foldable) for a human to pick up.
1431        //
1432        // On --resume the run gets one chance to repair itself: the seats a
1433        // rate limit took out are re-asked. If their quota has since reset and
1434        // the quorum is restored, the run picks up and finishes; otherwise it
1435        // stays stale and still-resumable for a later retry. If it does not
1436        // recover, the returned status stays `Stalled` and nothing was
1437        // clobbered (the recovery only mutates entries for the lost seats).
1438        if self.state.status == RunStatus::Stalled {
1439            if self.recover_stall().await? {
1440                self.finish_after_tally().await?;
1441            } else {
1442                // Still below quorum: persist the marker and stay resumable.
1443                self.state.save()?;
1444            }
1445            return Ok(());
1446        }
1447        // A run parked inside `land` - watching CI, mid fix-round, or
1448        // waiting on the owner's merge approval - resumes directly into it,
1449        // never back through `prep`. Everything before `merge` already
1450        // concluded; that is the only way `status` reaches `Landing` in the
1451        // first place. Re-walking `review_loop` first would also be actively
1452        // wrong: its own status recomputation (see its doc) treats any
1453        // clean round as reason to set `status` to `Gating`, which would
1454        // clobber this marker before `merge` ever ran, and this run would
1455        // never find its way back into `land` at all.
1456        if self.state.status == RunStatus::Landing
1457            || crate::handover::landing_checkpoint(&self.state)
1458        {
1459            self.state.land_sync_pending = true;
1460            self.state.save()?;
1461            self.run_land().await?;
1462            // `run_land` may have settled the run right here - CI came back
1463            // green and the PR merged, say - without ever passing back
1464            // through `merge`'s own trailing call. Whatever it left `status`
1465            // as is what this has to read.
1466            self.settle_questions();
1467            return Ok(());
1468        }
1469        // A run parked at `merge` on the owner's word about a withheld PR text
1470        // resumes straight into `merge`. Re-walking the tail would sync to the
1471        // base again (possibly rebasing, possibly spending the fixer) after the
1472        // gate had passed, and then push a tree nobody reviewed or gated.
1473        if self.state.github_text.is_some()
1474            && self.state.merge.is_none()
1475            && self.state.status == RunStatus::Gating
1476        {
1477            self.merge().await?;
1478            if self.resource_halt()? {
1479                return Ok(());
1480            }
1481            self.state.save()?;
1482            return Ok(());
1483        }
1484        if let Err(e) = self.prep().await {
1485            if e.is::<crate::repo_lock::ResourceBlocked>() {
1486                return Err(e);
1487            }
1488            self.state.status = RunStatus::Blocked;
1489            self.state
1490                .prep_failure
1491                .get_or_insert_with(|| format!("{e:#}"));
1492            self.state
1493                .event("prep", format!("preparation stopped: {e:#}"));
1494            self.state.save()?;
1495            return Err(e);
1496        }
1497        if self.park_here()? {
1498            return Ok(());
1499        }
1500        self.advise().await?;
1501        if self.park_here()? {
1502            return Ok(());
1503        }
1504        self.implement().await?;
1505        if self.park_here()? {
1506            return Ok(());
1507        }
1508        // `after_implement` already saved the state and settled any open
1509        // questions when it set this; nothing later in the graph has
1510        // anything to judge.
1511        if self.state.status == RunStatus::VerifiedNoop {
1512            return Ok(());
1513        }
1514        self.judge().await?;
1515        if self.park_here()? {
1516            return Ok(());
1517        }
1518        self.deliberate().await?;
1519        if self.park_here()? {
1520            return Ok(());
1521        }
1522        self.vote().await?;
1523        if self.park_here()? {
1524            return Ok(());
1525        }
1526        self.tally()?;
1527        // A verdict that lost its quorum is not trustworthy: do not review,
1528        // gate, or merge on it. Everything already done is kept, so the run
1529        // stays resumable (or foldable); the human can replace the agent that
1530        // ran out of quota and pick it up.
1531        if self.state.status == RunStatus::Stalled {
1532            // Persist the stalled marker now — the normal end-of-execute save
1533            // below is below this early return, and without it a resumed run
1534            // would reload a pre-tally status and keep going.
1535            self.state.save()?;
1536            return Ok(());
1537        }
1538        self.finish_after_tally().await?;
1539        Ok(())
1540    }
1541
1542    /// A wave could not start its agents because a shared resource stayed busy
1543    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1544    /// resumable. Unstarted seats have no result; completed peers retain
1545    /// their saved candidate state.
1546    fn stop_for_resource(&mut self) -> Result<()> {
1547        // A wait abandoned because the run is parking stays a park.
1548        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1549        {
1550            self.state.status = RunStatus::Blocked;
1551        }
1552        self.state.save()
1553    }
1554
1555    /// Stop walking the graph if a node ended on a resource block.
1556    fn resource_halt(&mut self) -> Result<bool> {
1557        if self.state.resource_block.is_none() {
1558            return Ok(false);
1559        }
1560        self.stop_for_resource()?;
1561        Ok(true)
1562    }
1563
1564    /// Park here if asked to, recording it in the run's own timeline.
1565    ///
1566    /// Returns whether the caller should stop walking the graph. The state is
1567    /// saved either way by the node that just finished; this adds the event so
1568    /// the operator's card says why a run that is neither finished nor moving
1569    /// is sitting where it is.
1570    fn park_here(&mut self) -> Result<bool> {
1571        let parking = self.pause.parked() || self.interrupt.parked();
1572        if !parking && self.state.resource_block.is_some() {
1573            self.stop_for_resource()?;
1574            return Ok(true);
1575        }
1576        // Either handle asking is enough - see `Pause`'s own doc for why
1577        // they are never the same one. `interrupt` is checked second so a
1578        // reason it carries is preferred in the message below over a plain
1579        // shutdown park racing it at the same boundary.
1580        if !self.pause.parked() && !self.interrupt.parked() {
1581            return Ok(false);
1582        }
1583        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1584        let why = match cause.as_ref().map(ParkCause::label) {
1585            Some(reason) => format!(
1586                "parked after `{}` ({reason}) — resume to carry on from here",
1587                self.state.status.as_str()
1588            ),
1589            None => format!(
1590                "parked after `{}` — resume to carry on from here",
1591                self.state.status.as_str()
1592            ),
1593        };
1594        self.state.event("park", why);
1595        self.state
1596            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1597        self.state.parked = true;
1598        self.state.save()?;
1599        Ok(true)
1600    }
1601
1602    /// Only a new explicit operator approval may retry failed preparation.
1603    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1604        if self.state.prep_failure.is_none() {
1605            return Ok(());
1606        }
1607        let failed_at = self
1608            .state
1609            .events
1610            .iter()
1611            .rev()
1612            .find(|event| event.node == "prep")
1613            .map_or(self.state.updated_at, |event| event.at);
1614        anyhow::ensure!(
1615            approval_at > failed_at,
1616            "preparation retry requires a new explicit release after the failure"
1617        );
1618        self.state.prep_failure = None;
1619        self.state.prep_complete = Some(false);
1620        self.state
1621            .event("prep-retry", "operator approved retry of preparation");
1622        Ok(())
1623    }
1624
1625    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1626    pub fn on_pause(&mut self, pause: Pause) {
1627        self.pause = pause;
1628    }
1629
1630    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1631    /// scheduler asking this one run - and no other - to park so a task
1632    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1633    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1634    /// handle.
1635    pub fn watch_interrupt(&mut self, pause: Pause) {
1636        self.interrupt = pause;
1637    }
1638
1639    /// Abandon this run's own open questions, once `status` has actually
1640    /// settled rather than merely paused.
1641    ///
1642    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1643    /// either back up with the candidates, the review round and the seat
1644    /// sessions already on disk, so a question an implementer asked mid-round
1645    /// may still get a real answer read by a real resume. Only the statuses
1646    /// `resumable` excludes are actually final: the run merged, it reached
1647    /// `Ready` with nothing left to do, it failed outright with no
1648    /// established point to continue from, or every candidate agreed, with
1649    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1650    /// every one of those the seat that asked is gone for good, exactly like
1651    /// the run being deleted under `magi run rm` - so the same cleanup
1652    /// applies, worded for what actually happened instead of "the run was
1653    /// deleted".
1654    ///
1655    /// Best-effort and silent on success: called from every place `status`
1656    /// can land on one of those three, including ones a resumed run revisits,
1657    /// so it must cost nothing when there was nothing open to begin with.
1658    fn settle_questions(&mut self) {
1659        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1660            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1661        }
1662    }
1663
1664    /// The tail of the graph after a trustworthy tally: fold losers, review,
1665    /// gate, merge, and persist.
1666    async fn finish_after_tally(&mut self) -> Result<()> {
1667        // Before the losers are folded: their branches are what the graft
1668        // reads from.
1669        self.graft().await?;
1670        if self.resource_halt()? {
1671            return Ok(());
1672        }
1673        self.fold_losers().await?;
1674        // Before review starts, and again right before the gate: a run's
1675        // review rounds can themselves take long enough for the base to move
1676        // a second time, and the gate is the one node whose "green" gets
1677        // acted on.
1678        self.sync_to_base().await?;
1679        if self.resource_halt()? {
1680            return Ok(());
1681        }
1682        if self.state.status == RunStatus::AlreadyInBase {
1683            return Ok(());
1684        }
1685        self.review_loop().await?;
1686        if self.resource_halt()? {
1687            return Ok(());
1688        }
1689        self.sync_to_base().await?;
1690        if self.resource_halt()? {
1691            return Ok(());
1692        }
1693        if self.state.status == RunStatus::AlreadyInBase {
1694            return Ok(());
1695        }
1696        self.gate().await?;
1697        self.merge().await?;
1698        if self.resource_halt()? {
1699            return Ok(());
1700        }
1701        self.state.save()?;
1702        Ok(())
1703    }
1704
1705    // ---------------------------------------------------------------- prep
1706
1707    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1708    /// run with the step and its output in the event; it is never swallowed.
1709    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1710    /// resume starts over instead of finding candidates and missing seats.
1711    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1712        let result = match ensure_setup_config(&mut self.state, repo).await {
1713            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1714            Err(e) => Err(e),
1715        };
1716        if let Err(e) = result {
1717            git::worktree_remove_clean(repo, wt).await.ok();
1718            if in_prep {
1719                self.rollback_prep(repo).await;
1720            }
1721            self.state.status = RunStatus::Blocked;
1722            self.state.event(
1723                "setup",
1724                format!("worktree setup failed in {}: {e:#}", wt.display()),
1725            );
1726            self.state.save()?;
1727            return Err(e);
1728        }
1729        Ok(())
1730    }
1731
1732    /// Undo everything `prep` created, so a resume re-runs it whole.
1733    async fn rollback_prep(&mut self, repo: &Path) {
1734        let root = self.state.worktree_root();
1735        for c in std::mem::take(&mut self.state.candidates) {
1736            if matches!(
1737                git::worktree_remove_clean(repo, &c.worktree).await,
1738                Ok(true)
1739            ) {
1740                git::branch_delete(repo, &c.branch).await.ok();
1741            } else {
1742                self.state.candidates.push(c);
1743            }
1744        }
1745        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1746        for j in 1..=seats {
1747            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1748                .await
1749                .ok();
1750        }
1751        for k in 1..=self.state.config.graph.advisors {
1752            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1753                .await
1754                .ok();
1755        }
1756    }
1757
1758    async fn prep(&mut self) -> Result<()> {
1759        if let Some(reason) = &self.state.prep_failure {
1760            bail!(
1761                "candidate preparation previously failed: {reason}; correct the source before retrying"
1762            );
1763        }
1764        if self
1765            .state
1766            .prep_complete
1767            .unwrap_or(!self.state.candidates.is_empty())
1768        {
1769            return Ok(());
1770        }
1771        let repo = self.state.repo.clone();
1772        // Includes rollback, config, refs, checkout creation and setup. No
1773        // repository mutation may precede the real shared lock acquisition.
1774        let _coordination = self.coordinate(&repo).await?;
1775        let base = self.state.base_commit.clone();
1776        let plan = self.state.sources.plan(&repo, &base).await?;
1777        self.rollback_prep(&repo).await;
1778        if !self.state.candidates.is_empty() {
1779            bail!(
1780                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1781            );
1782        }
1783        self.state.prep_complete = Some(false);
1784        self.state.status = RunStatus::Prep;
1785        self.state.save()?;
1786        let repo = self.state.repo.clone();
1787        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1788        let root = self.state.worktree_root();
1789        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1790
1791        // The hook is the write-time half of the blindness contract; the
1792        // presentation filter in `blind` is the half that cannot be bypassed.
1793        let hooks_dir = self.state.dir().join("hooks");
1794        if self.state.config.blind.commit_msg_hook {
1795            std::fs::create_dir_all(&hooks_dir)
1796                .with_context(|| format!("create {}", hooks_dir.display()))?;
1797            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1798            let path = hooks_dir.join("commit-msg");
1799            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1800            make_executable(&path)?;
1801            // Ref-counted rather than a plain idempotent set: with more than
1802            // one run able to be in flight in the same repository at once
1803            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1804            // true?" check cannot tell "another run of mine still needs
1805            // this" from "nobody does", and the run that happens to finish
1806            // first would disable the hook out from under a sibling still
1807            // relying on it.
1808            git::acquire_worktree_config(&repo).await?;
1809            self.state.enabled_worktree_config = true;
1810        }
1811
1812        // `[worktree] setup` hides its products per worktree, which needs the
1813        // same per-worktree config the hook does. Held until fold, like it.
1814        ensure_setup_config(&mut self.state, &repo).await?;
1815
1816        for (index, (spec, label)) in self
1817            .roles
1818            .implementers
1819            .clone()
1820            .into_iter()
1821            .zip(labels)
1822            .enumerate()
1823        {
1824            let branch = self.state.branch_for(label);
1825            let worktree = root.join(format!("cand-{label}"));
1826            // Persist intent before Git can leave a partially created tree.
1827            // Candidate registration follows setup, so it cannot inventory
1828            // crashes or refusals during creation itself.
1829            self.state.event(
1830                "prep-intent",
1831                serde_json::json!({
1832                    "branch": branch,
1833                    "worktree": worktree,
1834                    "start": start,
1835                    "picks": plan.picks,
1836                })
1837                .to_string(),
1838            );
1839            self.state.save()?;
1840            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1841            if self.state.config.blind.commit_msg_hook {
1842                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1843            }
1844            git::local_exclude(&worktree, "/.magi/").await?;
1845            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1846                // Nothing of this prep may survive: a resume must start over,
1847                // not find half the candidates and skip the rest.
1848                git::worktree_remove_clean(&repo, &worktree).await.ok();
1849                git::branch_delete(&repo, &branch).await.ok();
1850                self.rollback_prep(&repo).await;
1851                self.state.status = RunStatus::Blocked;
1852                self.state
1853                    .event("prep", format!("worktree setup failed: {e:#}"));
1854                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1855                self.state.save()?;
1856                return Err(e);
1857            }
1858            self.state.candidates.push(Candidate {
1859                index,
1860                label,
1861                agent: spec.id.clone(),
1862                branch,
1863                worktree,
1864                summary: String::new(),
1865                stat: String::new(),
1866                files: 0,
1867                commits: 0,
1868                empty: false,
1869                failed: None,
1870                verified_noop: None,
1871                duration_ms: 0,
1872                folded: false,
1873            });
1874            self.state.save()?;
1875            for pick in &plan.picks {
1876                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1877                {
1878                    self.state.status = RunStatus::Blocked;
1879                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1880                    self.state
1881                        .event("prep", format!("cannot apply referenced commit: {e}"));
1882                    self.state.save()?;
1883                    return Err(e);
1884                }
1885            }
1886        }
1887
1888        for j in 1..=self.roles.judges.len() {
1889            let wt = root.join(format!("judge-{j}"));
1890            if !wt.exists() {
1891                git::worktree_add_detached(&repo, &wt, &base).await?;
1892                self.setup_seat_worktree(&repo, &wt, true).await?;
1893            }
1894        }
1895
1896        // Disposable, detached checkouts for the design-deliberation stage's
1897        // advisor seats — the same shape as the judges' above, at the same
1898        // base commit, since advisors also only ever read. Sized off the
1899        // configured count directly rather than a resolved roster: unlike
1900        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1901        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1902        // `prep` has no `ResolvedRoles` field to read a count from here.
1903        if self.state.config.graph.advise {
1904            for k in 1..=self.state.config.graph.advisors {
1905                let wt = root.join(format!("advisor-{k}"));
1906                if !wt.exists() {
1907                    git::worktree_add_detached(&repo, &wt, &base).await?;
1908                    self.setup_seat_worktree(&repo, &wt, true).await?;
1909                }
1910            }
1911        }
1912
1913        // A judge cannot tell it is looking at its own patch — the seats keep
1914        // separate conversations — but a panel that shares agents with the
1915        // field is less independent than it looks, and that is worth saying out
1916        // loud once per run rather than leaving it in the config.
1917        let authors: Vec<&str> = self
1918            .roles
1919            .implementers
1920            .iter()
1921            .map(|a| a.id.as_str())
1922            .collect();
1923        let overlap: Vec<String> = self
1924            .roles
1925            .judges
1926            .iter()
1927            .enumerate()
1928            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1929            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1930            .collect();
1931        if !overlap.is_empty() {
1932            let note = format!(
1933                "{} also authored a candidate; blind, but the panel is less \
1934                 independent than {} distinct agents would be",
1935                overlap.join(", "),
1936                self.roles.judges.len()
1937            );
1938            self.state.event("prep", note);
1939        }
1940
1941        self.state.event(
1942            "prep",
1943            format!(
1944                "{} candidates, {} judges, base {} ({})",
1945                self.state.candidates.len(),
1946                self.roles.judges.len(),
1947                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1948                self.state.base_branch
1949            ),
1950        );
1951        self.state.prep_complete = Some(true);
1952        self.state.status = RunStatus::Implementing;
1953        self.state.save()?;
1954        Ok(())
1955    }
1956
1957    // -------------------------------------------------------------- advise
1958
1959    /// The design-deliberation stage: independent, read-only advisor seats
1960    /// each sketch a design before any implementer touches the repository,
1961    /// and (when at least one produced a usable proposal) a synthesis seat
1962    /// blends them into a brief `implement` carries in every candidate's
1963    /// prompt.
1964    ///
1965    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1966    /// advisors` is the proposal count. Everything here is best-effort and
1967    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1968    /// that cannot reach quota, or a synthesis seat that produced nothing
1969    /// usable all leave `implement` exactly as it was before this stage
1970    /// existed — the task instruction alone — rather than failing the whole
1971    /// competition over an enrichment stage. Every outcome is still recorded
1972    /// as an event, so a run that got nothing from this stage says why.
1973    ///
1974    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1975    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1976    /// resumed run whose stage failed would re-run it, and re-spend the
1977    /// agent calls, on every reentry before `implement`.
1978    ///
1979    /// Also skipped once any candidate shows implementation progress — the
1980    /// exact predicate `implement` itself uses to decide a candidate is no
1981    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1982    /// is not enough: a run created by an older binary that predates this
1983    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1984    /// an already-`Implementing`-or-later run under this build would
1985    /// otherwise walk straight back through `prep` (a no-op once candidates
1986    /// exist) into this node and spawn every advisor seat against worktrees
1987    /// `prep` never recreated — after implementation has already started,
1988    /// which is exactly the invariant this stage exists to guarantee.
1989    async fn advise(&mut self) -> Result<()> {
1990        let implement_untouched = self
1991            .state
1992            .candidates
1993            .iter()
1994            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1995        if !self.state.config.graph.advise || self.state.advise_attempted {
1996            return Ok(());
1997        }
1998        if !implement_untouched {
1999            self.state.event(
2000                "advise",
2001                "skipping the design-deliberation stage: at least one \
2002                 candidate already shows implementation progress, so this \
2003                 run is past the point the stage exists to run before"
2004                    .to_owned(),
2005            );
2006            self.state.advise_attempted = true;
2007            self.state.save()?;
2008            return Ok(());
2009        }
2010        let run_id = self.state.id.clone();
2011        let prompts = self.state.config.prompts.clone();
2012        let instruction = self.state.instruction.clone();
2013        let language = self.state.config.graph.language.clone();
2014        let root = self.state.worktree_root();
2015        let n = self.state.config.graph.advisors;
2016        let where_recorded = self.state.dir().join("run.json");
2017
2018        let seats = match self.state.config.advisors() {
2019            Ok(seats) if !seats.is_empty() => seats,
2020            Ok(_) => {
2021                self.state.event(
2022                    "advise",
2023                    format!(
2024                        "[graph] advisors is 0; skipping the design-deliberation \
2025                         stage and continuing without a synthesis brief (see {})",
2026                        where_recorded.display()
2027                    ),
2028                );
2029                self.state.advise_attempted = true;
2030                self.state.save()?;
2031                return Ok(());
2032            }
2033            Err(e) => {
2034                self.state.event(
2035                    "advise",
2036                    format!(
2037                        "could not resolve advisor seats ({e:#}); continuing \
2038                         without a design-deliberation brief (see {})",
2039                        where_recorded.display()
2040                    ),
2041                );
2042                self.state.advise_attempted = true;
2043                self.state.save()?;
2044                return Ok(());
2045            }
2046        };
2047
2048        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2049        let artifacts = agent::artifacts_dir(&self.state.dir());
2050        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2051
2052        let mut jobs = Vec::new();
2053        for (i, spec) in seats.iter().cloned().enumerate() {
2054            let seat_key = format!("advisor-{}", i + 1);
2055            let seat = self.seat(&seat_key, &spec.id);
2056            jobs.push(SeatJob {
2057                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2058                spec,
2059                seat,
2060                cwd: worktrees[i % worktrees.len()].clone(),
2061                timeout,
2062                allow_write: false,
2063                sessions: false,
2064                artifacts: artifacts.clone(),
2065                stem: seat_key,
2066                handover: None,
2067            });
2068        }
2069
2070        self.state.event(
2071            "advise",
2072            format!(
2073                "{} advisor seat(s) sketching a design in parallel",
2074                jobs.len()
2075            ),
2076        );
2077        let mut quota_losses = Vec::new();
2078        let cache = self.state.config.cache_dir();
2079        let ctx = WaveCtx {
2080            carry_seats: false,
2081            pauses: Some((&self.pause, &self.interrupt)),
2082            run: &run_id,
2083            node: "advise",
2084            prompts: &prompts,
2085            cache: cache.as_deref(),
2086            round: None,
2087        };
2088        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2089        let results = ask_json_wave::<Proposal>(
2090            jobs,
2091            Arc::clone(&self.sem),
2092            self.state.config.graph.retries,
2093            &advisor_roster,
2094            &ctx,
2095            &mut quota_losses,
2096            &mut self.state,
2097            &|p: &Proposal| p.validate(),
2098        )
2099        .await;
2100        if self.state.resource_block.is_some() {
2101            return self.stop_for_resource();
2102        }
2103        self.state.quota.extend(quota_losses);
2104
2105        let mut records = Vec::with_capacity(results.len());
2106        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2107            let agent_id = seat.agent.clone();
2108            self.state.seats.insert(seat.key.clone(), seat);
2109            match res {
2110                Ok((proposal, out)) => {
2111                    self.state
2112                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2113                    records.push(advise::AdvisorRecord::proposed(
2114                        i + 1,
2115                        agent_id,
2116                        proposal,
2117                        out.duration_ms,
2118                    ));
2119                }
2120                Err(e) => {
2121                    self.state.event(
2122                        "advise",
2123                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2124                    );
2125                    records.push(advise::AdvisorRecord::failed(
2126                        i + 1,
2127                        agent_id,
2128                        e.to_string(),
2129                    ));
2130                }
2131            }
2132        }
2133
2134        let mut advice = advise::Advice {
2135            records,
2136            synthesis: None,
2137        };
2138        if advice.proposals().is_empty() {
2139            self.state.event(
2140                "advise",
2141                "no advisor produced a usable proposal; continuing without a \
2142                 synthesis brief"
2143                    .to_owned(),
2144            );
2145        } else {
2146            match self
2147                .synthesize_brief(
2148                    &advice,
2149                    &instruction,
2150                    &language,
2151                    &worktrees[0],
2152                    &artifacts,
2153                    &run_id,
2154                    &prompts,
2155                    cache.as_deref(),
2156                )
2157                .await
2158            {
2159                Ok(Some(text)) => {
2160                    self.state.event(
2161                        "advise",
2162                        "synthesized a design brief for the implementer".to_owned(),
2163                    );
2164                    advice.synthesis = Some(text);
2165                }
2166                Ok(None) => {
2167                    self.state.event(
2168                        "advise",
2169                        "the synthesis seat produced nothing usable; continuing \
2170                         without a design brief"
2171                            .to_owned(),
2172                    );
2173                }
2174                Err(e) => {
2175                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2176                        return Err(e);
2177                    }
2178                    self.state.event(
2179                        "advise",
2180                        format!("could not synthesize a design brief: {e:#}"),
2181                    );
2182                }
2183            }
2184        }
2185        advise::apply_reflection(&mut advice);
2186
2187        self.state.advice = Some(advice);
2188        self.state.advise_attempted = true;
2189        self.state.save()?;
2190        Ok(())
2191    }
2192
2193    /// The synthesis seat: reads every advisor's proposal and blends them
2194    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2195    /// out of [`Runner::advise`] only for readability — it is not called
2196    /// anywhere else.
2197    ///
2198    /// Picked the same way [`crate::talk`]'s standing conversation and
2199    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2200    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2201    /// order (a claude seat, else the first runnable agent in roster order)
2202    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2203    /// in [`crate::config`] for why a dedicated field exists here at all.
2204    #[allow(clippy::too_many_arguments)]
2205    async fn synthesize_brief(
2206        &mut self,
2207        advice: &advise::Advice,
2208        instruction: &str,
2209        language: &str,
2210        cwd: &Path,
2211        artifacts: &Path,
2212        run_id: &str,
2213        prompts: &Prompts,
2214        cache: Option<&Path>,
2215    ) -> Result<Option<String>> {
2216        let chain = agent::pick_chain(
2217            &self.state.config.agents,
2218            self.state.config.roles.synthesizer.as_ref(),
2219            &agent::installed,
2220            "synthesizer",
2221        )?;
2222        let proposals = advice.proposals();
2223        let mut prompt = prompt::with_overlay(
2224            prompt::synthesize_brief(instruction, &proposals, language),
2225            prompts.overlay("advise"),
2226        );
2227        if cache.is_some() {
2228            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2229            // below — see `prompt::build_cache_note`'s doc for why telling a
2230            // read-only seat to build through the shared cache is exactly how
2231            // a sandbox's write refusal gets misread as a defect.
2232            prompt.push('\n');
2233            prompt.push_str(&prompt::build_cache_note("advise", false));
2234        }
2235        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2236        // Each id is tried once, in order; a quota hit, error or unusable
2237        // answer moves to the next. The seat is single-turn (`sessions:
2238        // false`) and the prompt is the whole context, so a fallback agent
2239        // needs nothing carried over.
2240        let mut last = None;
2241        for (n, spec) in chain.iter().enumerate() {
2242            if n > 0 {
2243                self.state
2244                    .event("advise", format!("synthesis falling back to {}", spec.id));
2245            }
2246            let mut seat = self.seat("advise-synthesis", &spec.id);
2247            let wait = self.repository_wait();
2248            let outcome = wait
2249                .scope(agent::invoke(
2250                    spec,
2251                    &mut seat,
2252                    &Invocation {
2253                        scope: crate::agent::InvocationScope::Repository,
2254                        coordination: self.state.config.coordination.as_ref(),
2255                        fetched_base: None,
2256                        cwd,
2257                        prompt: &prompt,
2258                        timeout,
2259                        allow_write: false,
2260                        unsandboxed: false,
2261                        sessions: false,
2262                        artifacts,
2263                        stem: &if n == 0 {
2264                            "advise-synthesis".to_owned()
2265                        } else {
2266                            format!("advise-synthesis-{}", spec.id)
2267                        },
2268                        run: run_id,
2269                        node: "advise",
2270                        cache_dir: None,
2271                        attachments: &[],
2272                        writable: &[],
2273                    },
2274                ))
2275                .await;
2276            if outcome.is_ok() {
2277                self.state.seats.insert(seat.key.clone(), seat);
2278            }
2279            let advance = agent::chain_advances(&outcome);
2280            last = Some(outcome);
2281            if !advance {
2282                break;
2283            }
2284        }
2285        // Exhausted: the last attempt's result is what a single failed seat
2286        // would have produced.
2287        let out = last.expect("a chain holds at least one agent")?;
2288        if !out.usable() {
2289            return Ok(None);
2290        }
2291        let text =
2292            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2293        Ok((!text.trim().is_empty()).then_some(text))
2294    }
2295
2296    // ----------------------------------------------------------- implement
2297
2298    async fn implement(&mut self) -> Result<()> {
2299        self.persist_candidates().await?;
2300        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2301        // agent files with `magi task add` name the run that paid for it. The
2302        // prompt overlay is cloned alongside it because the waves borrow it
2303        // while `self` is mutably borrowed by the node's own bookkeeping.
2304        let run_id = self.state.id.clone();
2305        let prompts = self.state.config.prompts.clone();
2306        let todo: Vec<usize> = self
2307            .state
2308            .candidates
2309            .iter()
2310            .enumerate()
2311            .filter(|(_, c)| {
2312                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2313                    && c.failed.is_none()
2314                    && !c.empty
2315            })
2316            .map(|(i, _)| i)
2317            .collect();
2318        if todo.is_empty() {
2319            return self.after_implement();
2320        }
2321        self.state.status = RunStatus::Implementing;
2322
2323        let language = self.state.config.graph.language.clone();
2324        let github_language = self.state.config.graph.github_language.clone();
2325        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2326        let sessions = self.state.config.graph.sessions;
2327        let artifacts = agent::artifacts_dir(&self.state.dir());
2328        // The design-deliberation stage's blended brief, when `advise` found
2329        // one — carried into every implementer's prompt the same way
2330        // regardless of which candidate it is.
2331        let brief = self
2332            .state
2333            .advice
2334            .as_ref()
2335            .and_then(|a| a.synthesis.as_deref())
2336            .map(str::to_owned);
2337        let attachments = self.state.attachments.clone();
2338
2339        let mut jobs = Vec::new();
2340        for &i in &todo {
2341            let (index, label, worktree) = {
2342                let c = &self.state.candidates[i];
2343                (c.index, c.label, c.worktree.clone())
2344            };
2345            let spec = self.roles.implementers[index].clone();
2346            let seat_key = format!("impl-{label}");
2347            let seat = self.seat(&seat_key, &spec.id);
2348            let instruction = seeded_instruction(&self.state);
2349            jobs.push(SeatJob {
2350                spec,
2351                seat,
2352                prompt: prompt::implement_in(
2353                    &instruction,
2354                    &worktree.to_string_lossy(),
2355                    &language,
2356                    &github_language,
2357                    brief.as_deref(),
2358                    &attachments,
2359                ),
2360                cwd: worktree,
2361                timeout,
2362                allow_write: true,
2363                sessions,
2364                artifacts: artifacts.clone(),
2365                stem: format!("impl-{label}"),
2366                handover: None,
2367            });
2368        }
2369
2370        self.state.event(
2371            "implement",
2372            format!("{} candidates in parallel", jobs.len()),
2373        );
2374        // Kept so a seat whose CLI hung up can be asked again from the same
2375        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2376        // can update a seat's own entry once a fallback agent takes it over —
2377        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2378        // whichever agent actually answered, not the one that quota'd out.
2379        let mut sent = jobs.clone();
2380        let cache = self.state.config.cache_dir();
2381        let ctx = WaveCtx {
2382            carry_seats: false,
2383            pauses: Some((&self.pause, &self.interrupt)),
2384            run: &run_id,
2385            node: "implement",
2386            prompts: &prompts,
2387            cache: cache.as_deref(),
2388            round: None,
2389        };
2390        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2391        if !wave_blocked(&results) {
2392            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2393                .await;
2394            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2395                .await?;
2396            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2397                .await;
2398        }
2399        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2400
2401        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2402            && results
2403                .iter()
2404                .any(|(_, _, out)| FailClass::of(out).is_some());
2405        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2406            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2407                // Executed turns interrupted by disk pressure already carry
2408                // a preservation barrier from wave. Admission-only waits do
2409                // not. Keep the interrupted seat's session for recovery too.
2410                if self
2411                    .state
2412                    .candidate_preservation
2413                    .contains_key(&self.state.candidates[i].label)
2414                {
2415                    self.state.seats.insert(seat.key.clone(), seat);
2416                }
2417                continue;
2418            }
2419            let seat_key = seat.key.clone();
2420            // A quota fallback (`resume_seat_handovers`) may have handed this
2421            // seat to a different agent than the one `prep` recorded on the
2422            // candidate; the stats tables and any later fixer-defaults-to-
2423            // winner's-author lookup must credit whoever actually answered —
2424            // unless every fallback also quota'd out, in which case nobody
2425            // actually answered and crediting the last agent tried would
2426            // erase every earlier agent's own quota loss from the stats
2427            // tables instead of just this one seat's.
2428            let agent = seat.agent.clone();
2429            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2430            self.state.seats.insert(seat.key.clone(), seat);
2431            let label = self.state.candidates[i].label;
2432            self.state.interrupted_implementations.remove(&label);
2433
2434            let (summary, duration, failed, verified_claim) = match out {
2435                AgentOutcome::Ok(o) => {
2436                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2437                    let failed = (!o.usable()).then(|| {
2438                        if o.timed_out {
2439                            "agent timed out".to_owned()
2440                        } else {
2441                            format!(
2442                                "agent exited with {}",
2443                                crate::notices::exit_status(o.exit_code)
2444                            )
2445                        }
2446                    });
2447                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2448                    (text, o.duration_ms, failed, verified_claim)
2449                }
2450                // Left un-resumed by `resume_undelivered` (a dirty tree
2451                // already rescues the work, or there was no session left to
2452                // resume into) — reported like the ordinary failure it is,
2453                // never as if `o.text` (the CLI's raw error JSON) were an
2454                // answer.
2455                AgentOutcome::Dropped(o) => {
2456                    let why = o
2457                        .dropped
2458                        .as_ref()
2459                        .map(|d| d.why.as_str())
2460                        .unwrap_or("the CLI ended the stream without delivering its answer");
2461                    (
2462                        String::new(),
2463                        o.duration_ms,
2464                        Some(format!("the CLI dropped the stream ({why})")),
2465                        None,
2466                    )
2467                }
2468                AgentOutcome::Quota(o) => {
2469                    self.state.quota.push(QuotaLoss {
2470                        seat: seat_key,
2471                        node: "implement".to_owned(),
2472                        at: Timestamp::now(),
2473                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2474                    });
2475                    (
2476                        String::new(),
2477                        o.duration_ms,
2478                        Some("rate limited (quota); produced no change".to_owned()),
2479                        None,
2480                    )
2481                }
2482                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2483                    (String::new(), 0, Some(e), None)
2484                }
2485            };
2486
2487            // Save every completed turn before any fallible Git operation. A failed
2488            // rescue must not erase the reply or prevent collection of other seats.
2489            let c = &mut self.state.candidates[i];
2490            if !exhausted_the_fallback_chain {
2491                c.agent = agent;
2492            }
2493            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2494            c.duration_ms = duration;
2495            c.failed = failed;
2496            c.verified_noop = verified_claim;
2497            self.state
2498                .candidate_preservation
2499                .insert(label, "awaiting persistence".to_owned());
2500        }
2501        self.state.save()?;
2502        self.persist_candidates().await?;
2503
2504        if resource_blocked {
2505            return self.stop_for_resource();
2506        }
2507        if selected_failed {
2508            self.state.status = RunStatus::Blocked;
2509            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2510            self.state.save()?;
2511            bail!(
2512                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2513            );
2514        }
2515        self.after_implement()
2516    }
2517
2518    /// Retry only persistence after a completed implementation turn.
2519    async fn persist_candidates(&mut self) -> Result<()> {
2520        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2521        for label in labels {
2522            let i = self
2523                .state
2524                .candidates
2525                .iter()
2526                .position(|c| c.label == label)
2527                .context("pending candidate is missing")?;
2528            let c = self.state.candidates[i].clone();
2529            let interrupted = self.state.interrupted_implementations.contains(&label);
2530            let result = async {
2531                let wait = crate::repo_lock::WaitPolicy::new(
2532                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2533                    || false,
2534                );
2535                let _coordination =
2536                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2537                // Best effort evidence requires no index writes. A successful snapshot
2538                // never authorizes removal of a tree whose rescue failed.
2539                if let Err(e) = git::snapshot_uncommitted(
2540                    &c.worktree,
2541                    &self
2542                        .state
2543                        .dir()
2544                        .join("artifacts")
2545                        .join(format!("candidate-{label}-recovery")),
2546                )
2547                .await
2548                {
2549                    self.state.event(
2550                        "preserve",
2551                        format!("candidate {label} snapshot incomplete: {e:#}"),
2552                    );
2553                }
2554                let rescued = git::rescue_commit(
2555                    &c.worktree,
2556                    &format!("magi: candidate {label} (uncommitted work)"),
2557                )
2558                .await?;
2559                self.state.note_withheld("implement", &rescued.withheld);
2560                let base = &self.state.base_commit;
2561                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2562                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2563                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2564                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2565                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2566                if c.empty && commits > 0 && !patch.trim().is_empty()
2567                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2568                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2569                        || self.state.tally.is_some())
2570                {
2571                    // Keep the old classification and pending marker so every
2572                    // retry stops too. Retain decisions as recovery evidence;
2573                    // only a supported recovery operation may replace them.
2574                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2575                }
2576                let c = &mut self.state.candidates[i];
2577                c.commits = commits;
2578                c.stat = stat;
2579                c.files = files;
2580                let no_changes = commits == 0 || patch.trim().is_empty();
2581                c.empty = no_changes && !interrupted;
2582                if interrupted && no_changes {
2583                    c.failed = None;
2584                    c.verified_noop = None;
2585                }
2586                if !c.empty {
2587                    c.failed = None;
2588                    c.verified_noop = None;
2589                }
2590                self.state.event(
2591                    "preserve",
2592                    format!("candidate {label}: {files} files, {commits} commits"),
2593                );
2594                Ok::<_, anyhow::Error>(())
2595            }
2596            .await;
2597            match result {
2598                Ok(()) => {
2599                    self.state.candidate_preservation.remove(&label);
2600                    if !interrupted || self.state.candidates[i].files > 0 {
2601                        self.state.interrupted_implementations.remove(&label);
2602                    }
2603                }
2604                Err(e) => {
2605                    let reason = format!("candidate {label} preservation failed: {e:#}");
2606                    self.state.event("preserve", &reason);
2607                    self.state.candidate_preservation.insert(label, reason);
2608                }
2609            }
2610            if self.state.candidate_preservation.is_empty()
2611                && self.state.status == RunStatus::Blocked
2612                && self.state.candidates.iter().all(|c| c.failed.is_none())
2613            {
2614                // Persist the recovered phase with the cleared pending marker;
2615                // a crash between saves must not turn this into a selected-seat
2616                // failure hold on the next resume.
2617                self.state.status = RunStatus::Implementing;
2618                self.state.event(
2619                    "preserve",
2620                    "candidate persistence recovered; implementation can continue",
2621                );
2622            }
2623            self.state.save()?;
2624        }
2625        if !self.state.candidate_preservation.is_empty() {
2626            self.state.status = RunStatus::Blocked;
2627            self.state.save()?;
2628            bail!(
2629                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2630            );
2631        }
2632        Ok(())
2633    }
2634
2635    /// Ask again, once, for work a CLI did and then failed to hand over.
2636    ///
2637    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2638    /// status with an empty response and a usage report showing output tokens,
2639    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2640    /// seven minutes and 14,267 output tokens that arrived as an empty
2641    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2642    ///
2643    /// Two conditions, and both matter:
2644    ///
2645    /// - **Only when the tree is untouched.** Often the agent has already
2646    ///   written its files and only the closing message was lost; the rescue
2647    ///   commit below picks that up and there is nothing to ask for. Re-asking
2648    ///   then would pay for a second implementation of work already on disk.
2649    /// - **Once.** A CLI that drops one stream can drop the next, and this
2650    ///   node is the most expensive in the graph.
2651    ///
2652    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2653    /// dropped reply still carried its `conversation_id`, so the seat is asked
2654    /// to finish what it was doing rather than sent the whole task again. It
2655    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2656    /// node's - for the same reason a re-ranked judge does: restating finished
2657    /// work is not the work.
2658    ///
2659    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2660    /// same way until it resets, while an abandoned conversation is still
2661    /// there to be picked up.
2662    async fn resume_undelivered(
2663        &mut self,
2664        results: &mut [(usize, SeatState, AgentOutcome)],
2665        sent: &[SeatJob],
2666        prompts: &Prompts,
2667        run_id: &str,
2668    ) {
2669        for (wi, seat, out) in results.iter_mut() {
2670            let Some(dropped) = (match &*out {
2671                AgentOutcome::Dropped(o) => o.dropped.clone(),
2672                _ => None,
2673            }) else {
2674                continue;
2675            };
2676            let Some(job) = sent.get(*wi) else { continue };
2677            // Already on disk? Then only the closing message was lost.
2678            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2679                self.state.event(
2680                    "implement",
2681                    format!(
2682                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2683                         work is in the tree",
2684                        seat.key, dropped.output_tokens, dropped.why
2685                    ),
2686                );
2687                continue;
2688            }
2689            // The re-ask only makes sense as a resume: `resume_after_drop`
2690            // says nothing about the task, trusting the seat to still hold it.
2691            // Without a session to resume — sessions disabled, or this CLI's
2692            // drop shape happened not to carry a session id — that prompt
2693            // would open a brand-new conversation with no context at all,
2694            // which is worse than leaving this as the ordinary failure it
2695            // already is.
2696            if !has_context(&job.spec, seat, job.sessions) {
2697                self.state.event(
2698                    "implement",
2699                    format!(
2700                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2701                         is no session left to resume",
2702                        seat.key, dropped.output_tokens, dropped.why
2703                    ),
2704                );
2705                continue;
2706            }
2707            self.state.event(
2708                "implement",
2709                format!(
2710                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2711                     conversation",
2712                    seat.key, dropped.output_tokens, dropped.why
2713                ),
2714            );
2715            let mut retry = job.clone();
2716            retry.seat = seat.clone();
2717            retry.prompt = prompt::resume_after_drop(&dropped.why);
2718            retry.timeout = retry_budget(job.timeout, true);
2719            retry.stem = format!("{}-resume", job.stem);
2720            let cache = self.state.config.cache_dir();
2721            let ctx = WaveCtx {
2722                carry_seats: false,
2723                pauses: Some((&self.pause, &self.interrupt)),
2724                run: run_id,
2725                node: "implement",
2726                prompts,
2727                cache: cache.as_deref(),
2728                round: None,
2729            };
2730            let (resumed_seat, resumed) =
2731                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2732            *seat = resumed_seat;
2733            *out = resumed;
2734        }
2735    }
2736
2737    /// Fall an implement seat through to the next untried agent in the
2738    /// implementer roster when it lost to quota — or, since the handover was
2739    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2740    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2741    /// quota path itself being unchanged — instead of leaving the
2742    /// seat's loss final the moment one agent's account runs dry.
2743    ///
2744    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2745    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2746    /// truncates to the single slot rotation picked, so a solo task whose one
2747    /// implementer hits quota mid-run used to have nothing else to try. This
2748    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2749    /// unrotated roster — which is the only place the *other* candidates in
2750    /// the machine's roster still exist once `implementers` has been cut down
2751    /// to size.
2752    ///
2753    /// Walks forward from just past the seat's own original position in the
2754    /// roster, never wrapping back to the front: a later candidate slot (say
2755    /// `beta`, the roster's second entry) must fall through to the *next*
2756    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2757    /// almost certainly a different candidate's own agent already — and once
2758    /// the roster's tail is exhausted there is nothing left to fall through
2759    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2760    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2761    /// let this retry that id forever. The loop keeps falling through until
2762    /// an attempt lands something other than `Quota` or the roster's tail
2763    /// runs out of untried ids, at which point the seat is left exactly as
2764    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2765    /// `QuotaLoss` recorded, the candidate failed/empty.
2766    ///
2767    /// `sent` is taken mutably and updated with the fallback agent's spec:
2768    /// `resume_unconfirmed_commands`, which runs after this and also reads
2769    /// `sent`, must see whichever agent actually ended up answering the seat
2770    /// — reading the stale, original spec there would check session
2771    /// eligibility against the wrong CLI and could hand a fallback agent's
2772    /// session id to the agent that just lost the seat to quota.
2773    ///
2774    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2775    /// — `self.seat` only reuses state when the agent id is unchanged, so
2776    /// handing it a different id already gets this for free. Reusing the old
2777    /// seat would resume a different CLI's session as if it were a
2778    /// continuation of this one.
2779    ///
2780    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2781    /// a quota loss cuts an agent off mid-turn, so anything already in the
2782    /// tree is unfinished work, not a completed candidate a re-ask would pay
2783    /// for twice. A dirty tree is rescued into a commit first (the same
2784    /// neutral-identity rescue `implement`'s own outcome loop gives every
2785    /// candidate) so the next agent starts clean.
2786    ///
2787    /// The new agent gets the implementer's full prompt and full
2788    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2789    /// it has no session and no context, and is implementing the task from
2790    /// nothing, unlike a resumed drop which is only restating work already
2791    /// done.
2792    ///
2793    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2794    /// `implement` event, never into `self.state.quota` — that is what
2795    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2796    /// go unspent, and a seat that ultimately recovered on its second or
2797    /// third agent is not the stalled panel that check exists to catch. Only
2798    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2799    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2800    /// outcome loop already has — this helper never pushes to it itself.
2801    async fn resume_seat_handovers(
2802        &mut self,
2803        results: &mut [(usize, SeatState, AgentOutcome)],
2804        sent: &mut [SeatJob],
2805        prompts: &Prompts,
2806        run_id: &str,
2807    ) -> Result<()> {
2808        let instruction = seeded_instruction(&self.state);
2809        let language = self.state.config.graph.language.clone();
2810        let github_language = self.state.config.graph.github_language.clone();
2811        let brief = self
2812            .state
2813            .advice
2814            .as_ref()
2815            .and_then(|a| a.synthesis.as_deref())
2816            .map(str::to_owned);
2817        let attachments = self.state.attachments.clone();
2818        for (wi, seat, out) in results.iter_mut() {
2819            // Who holds the other candidate seats of this wave right now
2820            // (earlier handovers already written back to `sent`).
2821            let others: BTreeSet<String> = sent
2822                .iter()
2823                .enumerate()
2824                .filter(|(j, _)| j != wi)
2825                .map(|(_, j)| j.spec.id.clone())
2826                .collect();
2827            let Some(job) = sent.get_mut(*wi) else {
2828                continue;
2829            };
2830            // Where the seat's own original agent sits in the roster — the
2831            // fallback walk starts just past here, never at the front, so a
2832            // later candidate slot's quota loss does not fall back onto an
2833            // earlier slot's own agent.
2834            let start = self
2835                .roles
2836                .implementer_roster
2837                .iter()
2838                .position(|s| s.id == job.spec.id)
2839                .unwrap_or(0);
2840            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2841            let mut fallback_attempt = 0usize;
2842            let mut prev: Option<FailClass> = None;
2843            while let Some(cur) = FailClass::of(&*out) {
2844                if !should_hand_over(prev.as_ref(), &cur) {
2845                    break;
2846                }
2847                let Some(next) =
2848                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2849                        .cloned()
2850                else {
2851                    break;
2852                };
2853                tried.insert(next.id.clone());
2854                fallback_attempt += 1;
2855
2856                let coordination = self.coordinate(&job.cwd).await?;
2857                if let Ok(r) = git::rescue_commit(
2858                    &job.cwd,
2859                    &format!(
2860                        "magi: candidate {} (uncommitted work before {} fallback)",
2861                        seat.key,
2862                        if cur == FailClass::Quota {
2863                            "quota"
2864                        } else {
2865                            "handover"
2866                        }
2867                    ),
2868                )
2869                .await
2870                {
2871                    self.state.note_withheld("implement", &r.withheld);
2872                }
2873                drop(coordination);
2874
2875                record_handover(
2876                    &mut self.state,
2877                    "implement",
2878                    &seat.key,
2879                    &seat.agent,
2880                    &next.id,
2881                    &cur,
2882                    &fail_reason(&*out),
2883                );
2884                prev = Some(cur.clone());
2885
2886                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2887                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2888                // Kept in sync on `sent` itself, not just the local retry: a
2889                // later helper (`resume_unconfirmed_commands`) reads `sent`
2890                // after this one returns and must see whichever agent is now
2891                // occupying the seat, not the one that just quota'd out —
2892                // otherwise it would judge session/continuation eligibility
2893                // by the wrong CLI and could resend a fallback's session id
2894                // to the agent that lost it the seat in the first place.
2895                job.spec = next.clone();
2896                let mut retry = job.clone();
2897                retry.seat = new_seat;
2898                retry.prompt = prompt::implement_in(
2899                    &instruction,
2900                    &job.cwd.to_string_lossy(),
2901                    &language,
2902                    &github_language,
2903                    brief.as_deref(),
2904                    &attachments,
2905                );
2906                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2907                let cache = self.state.config.cache_dir();
2908                let ctx = WaveCtx {
2909                    carry_seats: false,
2910                    pauses: Some((&self.pause, &self.interrupt)),
2911                    run: run_id,
2912                    node: "implement",
2913                    prompts,
2914                    cache: cache.as_deref(),
2915                    round: None,
2916                };
2917                let (fallback_seat, fallback_out) = run_one(
2918                    retry,
2919                    Arc::clone(&self.sem),
2920                    &ctx,
2921                    &mut self.state,
2922                    fallback_attempt,
2923                )
2924                .await;
2925                *seat = fallback_seat;
2926                *out = fallback_out;
2927            }
2928        }
2929        Ok(())
2930    }
2931
2932    /// Ask an implement seat's own CLI to confirm what it started, once, when
2933    /// its reply reported a command whose completion status it never
2934    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2935    /// that does and does not mean.
2936    ///
2937    /// The completion contract this task asks for, extended to `implement`
2938    /// with the same signal `continue_fix_report` reads for the fixer,
2939    /// rather than a keyword search over the reply or a hard requirement on
2940    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2941    /// a candidate's CLI turn ended cleanly while a test run it had started
2942    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2943    /// named in it at all is untouched by this: `commands` is empty, so
2944    /// there is nothing to be unconfirmed.
2945    ///
2946    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2947    /// this is not about recovering edits that might already be on disk, it
2948    /// is about a result the seat itself never vouched for, which resuming
2949    /// asks for regardless of what the tree already holds. Bounded to one
2950    /// attempt for the same reason `resume_undelivered` is — this is the
2951    /// most expensive node in the graph — and a seat that still cannot
2952    /// confirm on that attempt is left as whatever its (possibly still
2953    /// unconfirmed) reply says; this does not invent a new "failed" reason
2954    /// for a candidate that otherwise produced a real, committed change.
2955    async fn resume_unconfirmed_commands(
2956        &mut self,
2957        results: &mut [(usize, SeatState, AgentOutcome)],
2958        sent: &[SeatJob],
2959        prompts: &Prompts,
2960        run_id: &str,
2961    ) {
2962        for (wi, seat, out) in results.iter_mut() {
2963            let AgentOutcome::Ok(o) = &*out else {
2964                continue;
2965            };
2966            if !has_unconfirmed_command(&o.commands) {
2967                continue;
2968            }
2969            let Some(job) = sent.get(*wi) else { continue };
2970            if !has_context(&job.spec, seat, job.sessions) {
2971                self.state.event(
2972                    "implement",
2973                    format!(
2974                        "{}: the reply named a command whose own CLI never confirmed the exit \
2975                         status of, but there is no session left to resume",
2976                        seat.key
2977                    ),
2978                );
2979                continue;
2980            }
2981            self.state.event(
2982                "implement",
2983                format!(
2984                    "{}: the reply named a command whose own CLI never confirmed the exit \
2985                     status of; resuming the conversation",
2986                    seat.key
2987                ),
2988            );
2989            let mut retry = job.clone();
2990            retry.seat = seat.clone();
2991            retry.prompt = prompt::resume_incomplete(
2992                "a command in your last reply had no confirmed exit status",
2993            );
2994            retry.timeout = retry_budget(job.timeout, true);
2995            retry.stem = format!("{}-confirm", job.stem);
2996            let cache = self.state.config.cache_dir();
2997            let ctx = WaveCtx {
2998                carry_seats: false,
2999                pauses: Some((&self.pause, &self.interrupt)),
3000                run: run_id,
3001                node: "implement",
3002                prompts,
3003                cache: cache.as_deref(),
3004                round: None,
3005            };
3006            let (resumed_seat, resumed) =
3007                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3008            *seat = resumed_seat;
3009            *out = resumed;
3010        }
3011    }
3012
3013    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3014    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3015    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3016    /// that motivated this.
3017    ///
3018    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3019    /// own nudge loop already covers for judge/review/vote seats, and not a
3020    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3021    /// implement seats: here the CLI turn genuinely finished while the node's
3022    /// own work — the fixer's account of what it did — had not. Gated purely
3023    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3024    /// reply, never on any wording in it, so a fixer whose valid, first-try
3025    /// `FixReport` happens to mention having waited on a background test is
3026    /// never resumed — the `Ok(report)` branch at the call site returns
3027    /// before this is ever invoked.
3028    ///
3029    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3030    /// attempt ([`retry_budget`]), nothing attempted once the session is
3031    /// gone, and a quota hit ends the loop immediately rather than retrying a
3032    /// rate limit that fails the same way again.
3033    async fn continue_fix_report(
3034        &mut self,
3035        mut seat: SeatState,
3036        parse_err: String,
3037        job: &SeatJob,
3038        prompts: &Prompts,
3039        run_id: &str,
3040        round: usize,
3041    ) -> (
3042        SeatState,
3043        Option<FixReport>,
3044        Option<String>,
3045        ContinuationRecord,
3046    ) {
3047        let mut last_err = parse_err;
3048        let mut cumulative_wait_ms = 0u64;
3049        let mut attempts = 0usize;
3050        loop {
3051            if !has_context(&job.spec, &seat, job.sessions) {
3052                self.state.event(
3053                    "fix",
3054                    format!(
3055                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3056                         session left to resume into"
3057                    ),
3058                );
3059                let outcome = if attempts == 0 {
3060                    ContinuationOutcome::NoSession
3061                } else {
3062                    ContinuationOutcome::Exhausted
3063                };
3064                return (
3065                    seat,
3066                    None,
3067                    Some(format!("unparsable fix report: {last_err}")),
3068                    ContinuationRecord {
3069                        attempts,
3070                        cumulative_wait_ms,
3071                        outcome,
3072                    },
3073                );
3074            }
3075            if attempts >= MAX_FIX_CONTINUATIONS {
3076                self.state.event(
3077                    "fix",
3078                    format!(
3079                        "round {round}: fixer's reply still had no adoption report after \
3080                         {attempts} continuation(s) ({last_err}); giving up"
3081                    ),
3082                );
3083                return (
3084                    seat,
3085                    None,
3086                    Some(format!(
3087                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3088                    )),
3089                    ContinuationRecord {
3090                        attempts,
3091                        cumulative_wait_ms,
3092                        outcome: ContinuationOutcome::Exhausted,
3093                    },
3094                );
3095            }
3096            attempts += 1;
3097            self.state.event(
3098                "fix",
3099                format!(
3100                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3101                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3102                ),
3103            );
3104            let mut retry = job.clone();
3105            retry.seat = seat.clone();
3106            retry.prompt = prompt::resume_incomplete(&last_err);
3107            retry.timeout = retry_budget(job.timeout, true);
3108            retry.stem = format!("{}-continue{attempts}", job.stem);
3109            let cache = self.state.config.cache_dir();
3110            let ctx = WaveCtx {
3111                carry_seats: false,
3112                pauses: Some((&self.pause, &self.interrupt)),
3113                run: run_id,
3114                node: "fix",
3115                prompts,
3116                cache: cache.as_deref(),
3117                round: Some(round),
3118            };
3119            let (resumed_seat, resumed_out) = run_one(
3120                retry,
3121                Arc::clone(&self.sem),
3122                &ctx,
3123                &mut self.state,
3124                attempts,
3125            )
3126            .await;
3127            seat = resumed_seat;
3128            match resumed_out {
3129                AgentOutcome::Ok(o) => {
3130                    cumulative_wait_ms += o.duration_ms;
3131                    match verdict::extract_json::<FixReport>(&o.text) {
3132                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3133                            self.state.event(
3134                                "fix",
3135                                format!(
3136                                    "round {round}: fixer's adoption report recovered after \
3137                                     {attempts} continuation(s)"
3138                                ),
3139                            );
3140                            return (
3141                                seat,
3142                                Some(report),
3143                                None,
3144                                ContinuationRecord {
3145                                    attempts,
3146                                    cumulative_wait_ms,
3147                                    outcome: ContinuationOutcome::Resumed,
3148                                },
3149                            );
3150                        }
3151                        // The report parsed, but this same reply's own
3152                        // CommandEvidence — the identical record `state.jobs`
3153                        // renders — names a command whose CLI never
3154                        // confirmed an exit status. Read together, that is
3155                        // not a resolved answer: keep nudging rather than
3156                        // accept a report standing next to a command the
3157                        // seat's own CLI cannot vouch for.
3158                        Ok(_) => {
3159                            last_err = "the reply parsed, but it reported a command whose own CLI \
3160                                 never confirmed an exit status"
3161                                .to_owned();
3162                        }
3163                        Err(e) => last_err = e.to_string(),
3164                    }
3165                }
3166                AgentOutcome::Quota(o) => {
3167                    cumulative_wait_ms += o.duration_ms;
3168                    self.state.quota.push(QuotaLoss {
3169                        seat: seat.key.clone(),
3170                        node: "fix".to_owned(),
3171                        at: Timestamp::now(),
3172                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3173                    });
3174                    self.state.event(
3175                        "fix",
3176                        format!(
3177                            "round {round}: continuation rate limited (quota); not retrying now"
3178                        ),
3179                    );
3180                    return (
3181                        seat,
3182                        None,
3183                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3184                        ContinuationRecord {
3185                            attempts,
3186                            cumulative_wait_ms,
3187                            outcome: ContinuationOutcome::QuotaLost,
3188                        },
3189                    );
3190                }
3191                AgentOutcome::Dropped(o) => {
3192                    cumulative_wait_ms += o.duration_ms;
3193                    let why = o
3194                        .dropped
3195                        .as_ref()
3196                        .map(|d| d.why.as_str())
3197                        .unwrap_or("the CLI ended the stream without delivering its answer");
3198                    last_err = format!("the CLI dropped the stream ({why})");
3199                }
3200                AgentOutcome::Failed(e) => last_err = e,
3201                // The continuation was never started; the report stays lost
3202                // and the caller records that, without any handover.
3203                AgentOutcome::ResourceBlocked(e) => {
3204                    return (
3205                        seat,
3206                        None,
3207                        Some(format!("the continuation was not started: {e}")),
3208                        ContinuationRecord {
3209                            attempts: attempts.saturating_sub(1),
3210                            cumulative_wait_ms,
3211                            outcome: ContinuationOutcome::NoSession,
3212                        },
3213                    );
3214                }
3215            }
3216        }
3217    }
3218
3219    fn after_implement(&mut self) -> Result<()> {
3220        if !self.state.candidate_preservation.is_empty() {
3221            bail!("candidate preservation incomplete; judging is unsafe");
3222        }
3223        // Scan every candidate patch once the set is complete.
3224        if self.state.leaks.is_empty() {
3225            let cfg = self.state.config.blind.clone();
3226            let mut leaks = Vec::new();
3227            for c in &self.state.candidates {
3228                let Some(patch) =
3229                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3230                else {
3231                    continue;
3232                };
3233                leaks.extend(blind::scan(
3234                    &format!("candidate {} patch", c.label),
3235                    &patch,
3236                    &cfg.vendor_tokens,
3237                ));
3238            }
3239            if !leaks.is_empty() {
3240                let summary = leaks
3241                    .iter()
3242                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3243                    .collect::<Vec<_>>()
3244                    .join(", ");
3245                match cfg.on_leak {
3246                    LeakPolicy::Fail => {
3247                        self.state.status = RunStatus::Failed;
3248                        self.state
3249                            .event("blind", format!("vendor text in a patch: {summary}"));
3250                        self.state.leaks = leaks;
3251                        self.state.save()?;
3252                        self.settle_questions();
3253                        bail!(
3254                            "blind.on_leak = \"fail\" and vendor text reached a \
3255                             judged patch: {summary}"
3256                        );
3257                    }
3258                    LeakPolicy::Redact => self.state.event(
3259                        "blind",
3260                        format!("redacting vendor text for judging: {summary}"),
3261                    ),
3262                    LeakPolicy::Warn => self.state.event(
3263                        "blind",
3264                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3265                    ),
3266                }
3267                self.state.leaks = leaks;
3268            }
3269        }
3270
3271        if self.state.viable().is_empty() {
3272            if self.state.all_candidates_verified_noop() {
3273                // Every candidate agreed, with evidence the adoption guard
3274                // accepted, that nothing belongs in this worktree. That is
3275                // not the same fact as a candidate that simply failed to
3276                // write anything, and settling it as an ordinary `Failed`
3277                // (see `SCHEMA`'s doc for schema 10) is what let two of
3278                // task 391f's attempts burn a retry each re-discovering the
3279                // same already-landed fix. Terminal either way, so `judge`
3280                // must never run over an empty candidate set — unlike the
3281                // `Failed` branch below this returns `Ok`, not an error:
3282                // nothing here failed.
3283                self.state.status = RunStatus::VerifiedNoop;
3284                self.state.save()?;
3285                self.settle_questions();
3286                return Ok(());
3287            }
3288            self.state.status = RunStatus::Failed;
3289            self.state.save()?;
3290            self.settle_questions();
3291            bail!("no candidate produced a change; nothing to judge");
3292        }
3293        self.state.status = RunStatus::Judging;
3294        self.state.save()?;
3295        Ok(())
3296    }
3297
3298    // --------------------------------------------------------------- judge
3299
3300    async fn judge(&mut self) -> Result<()> {
3301        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3302        // agent files with `magi task add` name the run that paid for it. The
3303        // prompt overlay is cloned alongside it because the waves borrow it
3304        // while `self` is mutably borrowed by the node's own bookkeeping.
3305        let run_id = self.state.id.clone();
3306        let prompts = self.state.config.prompts.clone();
3307        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3308            return Ok(());
3309        }
3310        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3311        if viable.len() == 1 {
3312            // Recorded so this is a one-time event: `judgements` stays empty
3313            // either way, which without this flag is indistinguishable from
3314            // "not yet judged" on the next reentry — and status is left
3315            // untouched, so a later node's conclusion (e.g. `Blocked` after
3316            // the review budget ran out) survives a resume instead of being
3317            // clobbered back to `Judging` by this node running again.
3318            self.state.judge_skipped = true;
3319            self.state.event(
3320                "judge",
3321                format!(
3322                    "only candidate {} produced a change; judging skipped",
3323                    viable[0].label
3324                ),
3325            );
3326            self.state.save()?;
3327            return Ok(());
3328        }
3329        self.state.status = RunStatus::Judging;
3330
3331        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3332        let language = self.state.config.graph.language.clone();
3333        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3334        let sessions = self.state.config.graph.sessions;
3335        let artifacts = agent::artifacts_dir(&self.state.dir());
3336        let root = self.state.worktree_root();
3337        let base_short = short(&self.state.base_commit);
3338
3339        let mut jobs = Vec::new();
3340        let mut orders = Vec::new();
3341        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3342            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3343            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3344            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3345            let seat_key = format!("judge-{}", j + 1);
3346            let seat = self.seat(&seat_key, &spec.id);
3347            jobs.push(SeatJob {
3348                prompt: prompt::judge(
3349                    &self.state.instruction,
3350                    &views,
3351                    self.roles.judges.len(),
3352                    &base_short,
3353                    &language,
3354                ),
3355                spec,
3356                seat,
3357                cwd: root.join(format!("judge-{}", j + 1)),
3358                timeout,
3359                allow_write: false,
3360                sessions,
3361                artifacts: artifacts.clone(),
3362                stem: format!("judge-{}", j + 1),
3363                handover: None,
3364            });
3365        }
3366
3367        self.state.event(
3368            "judge",
3369            format!(
3370                "{} judges ranking {} candidates blind",
3371                jobs.len(),
3372                viable.len()
3373            ),
3374        );
3375        let labels_for_check = labels.clone();
3376        let mut quota_losses = Vec::new();
3377        let cache = self.state.config.cache_dir();
3378        let ctx = WaveCtx {
3379            carry_seats: false,
3380            pauses: Some((&self.pause, &self.interrupt)),
3381            run: &run_id,
3382            node: "judge",
3383            prompts: &prompts,
3384            cache: cache.as_deref(),
3385            round: None,
3386        };
3387        let results = ask_json_wave::<Ranking>(
3388            jobs,
3389            Arc::clone(&self.sem),
3390            self.state.config.graph.retries,
3391            &self.roles.judge_roster,
3392            &ctx,
3393            &mut quota_losses,
3394            &mut self.state,
3395            &move |r: &Ranking| r.validate(&labels_for_check),
3396        )
3397        .await;
3398        if self.state.resource_block.is_some() {
3399            return self.stop_for_resource();
3400        }
3401        self.state.quota.extend(quota_losses);
3402
3403        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3404            let agent_id = seat.agent.clone();
3405            self.state.seats.insert(seat.key.clone(), seat);
3406            let mut record = Judgement {
3407                judge: j + 1,
3408                seat: format!("judge-{}", j + 1),
3409                agent: agent_id,
3410                ranking: Vec::new(),
3411                reasons: BTreeMap::new(),
3412                confidence: None,
3413                strengths: BTreeMap::new(),
3414                order: orders[j].clone(),
3415                failed: None,
3416                duration_ms: 0,
3417            };
3418            match res {
3419                Ok((ranking, out)) => {
3420                    record.ranking = ranking.normalized();
3421                    record.reasons = ranking.reasons;
3422                    record.strengths = ranking.strengths;
3423                    record.confidence = ranking.confidence;
3424                    record.duration_ms = out.duration_ms;
3425                    self.state.event(
3426                        "judge",
3427                        format!(
3428                            "judge {} ranked {}",
3429                            j + 1,
3430                            record.ranking.iter().collect::<String>()
3431                        ),
3432                    );
3433                }
3434                Err(e) => {
3435                    record.failed = Some(e.to_string());
3436                    self.state
3437                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3438                }
3439            }
3440            self.state.judgements.push(record);
3441            self.state.save()?;
3442        }
3443        Ok(())
3444    }
3445
3446    // ---------------------------------------------------------- deliberate
3447
3448    async fn deliberate(&mut self) -> Result<()> {
3449        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3450        // agent files with `magi task add` name the run that paid for it. The
3451        // prompt overlay is cloned alongside it because the waves borrow it
3452        // while `self` is mutably borrowed by the node's own bookkeeping.
3453        let run_id = self.state.id.clone();
3454        let prompts = self.state.config.prompts.clone();
3455        if !self.state.deliberation.is_empty() {
3456            return Ok(());
3457        }
3458        let tops: Vec<char> = self
3459            .state
3460            .judgements
3461            .iter()
3462            .filter_map(|j| j.ranking.first().copied())
3463            .collect();
3464        let rounds = self.state.config.graph.deliberate_rounds;
3465        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3466            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3467                self.state.event(
3468                    "deliberate",
3469                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3470                );
3471            }
3472            self.state.status = RunStatus::Voting;
3473            self.state.save()?;
3474            return Ok(());
3475        }
3476
3477        self.state.status = RunStatus::Deliberating;
3478        self.state.event(
3479            "deliberate",
3480            format!(
3481                "split: first choices were {} — opening {rounds} round(s)",
3482                tops.iter().collect::<String>()
3483            ),
3484        );
3485
3486        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3487        let language = self.state.config.graph.language.clone();
3488        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3489        let sessions = self.state.config.graph.sessions;
3490        let artifacts = agent::artifacts_dir(&self.state.dir());
3491        let root = self.state.worktree_root();
3492        let base_short = short(&self.state.base_commit);
3493
3494        // Judges argue in sequence so that a turn can answer the one before it;
3495        // that is the difference between deliberation and three parallel
3496        // monologues.
3497        for round in 1..=rounds {
3498            let mut turns: Vec<DeliberationTurn> = Vec::new();
3499            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3500                if self.state.judgements[j].failed.is_some() {
3501                    continue;
3502                }
3503                let seat_key = format!("judge-{}", j + 1);
3504                let spec = self.occupant(&seat_key, spec);
3505                let mut seat = self.seat(&seat_key, &spec.id);
3506                let transcript = self.transcript(&turns, j);
3507                let build = |context: Option<&str>| {
3508                    prompt::deliberate(
3509                        &self.state.instruction,
3510                        context,
3511                        &transcript,
3512                        round,
3513                        rounds,
3514                        &language,
3515                    )
3516                };
3517                let block = self.candidate_block(&viable, &base_short);
3518                let full = build(Some(&block));
3519                let text = if has_context(&spec, &seat, sessions) {
3520                    build(None)
3521                } else {
3522                    full.clone()
3523                };
3524                let job = SeatJob {
3525                    spec,
3526                    seat: seat.clone(),
3527                    prompt: text,
3528                    cwd: root.join(format!("judge-{}", j + 1)),
3529                    timeout,
3530                    allow_write: false,
3531                    sessions,
3532                    artifacts: artifacts.clone(),
3533                    stem: format!("delib-{round}-judge-{}", j + 1),
3534                    handover: Some(full),
3535                };
3536                let cache = self.state.config.cache_dir();
3537                let ctx = WaveCtx {
3538                    carry_seats: false,
3539                    pauses: Some((&self.pause, &self.interrupt)),
3540                    run: &run_id,
3541                    node: "deliberate",
3542                    prompts: &prompts,
3543                    cache: cache.as_deref(),
3544                    round: None,
3545                };
3546                // A turn is never nudged (`retries` 0): a failed seat is
3547                // handed to the next roster agent, which gets the full
3548                // context. An empty answer is a turn, not a failure.
3549                let mut losses = Vec::new();
3550                let mut results = ask_wave_with::<String>(
3551                    vec![job],
3552                    Arc::clone(&self.sem),
3553                    0,
3554                    &self.roles.judge_roster,
3555                    &ctx,
3556                    &mut losses,
3557                    &mut self.state,
3558                    &|text: &str| {
3559                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3560                    },
3561                )
3562                .await;
3563                self.state.quota.extend(losses);
3564                let (updated, res, _) = results.pop().expect("one job in, one result out");
3565                seat = updated;
3566                let agent_id = seat.agent.clone();
3567                self.state.seats.insert(seat.key.clone(), seat);
3568                let body = match res {
3569                    Ok((body, _)) => body,
3570                    // Skip the seat; a CLI's raw error JSON is never read as
3571                    // this judge's position.
3572                    Err(e) => {
3573                        self.state
3574                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3575                        continue;
3576                    }
3577                };
3578                let tentative = verdict::extract_json::<Position>(&body)
3579                    .ok()
3580                    .and_then(|p| p.tentative)
3581                    .and_then(|s| s.trim().chars().next())
3582                    .map(|c| c.to_ascii_uppercase());
3583                self.state.event(
3584                    "deliberate",
3585                    format!(
3586                        "round {round}: judge {} now favours {}",
3587                        j + 1,
3588                        tentative.map_or("—".to_owned(), |c| c.to_string())
3589                    ),
3590                );
3591                turns.push(DeliberationTurn {
3592                    judge: j + 1,
3593                    agent: agent_id,
3594                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3595                    tentative,
3596                });
3597            }
3598            self.state
3599                .deliberation
3600                .push(DeliberationRound { round, turns });
3601            self.state.save()?;
3602        }
3603
3604        self.state.status = RunStatus::Voting;
3605        self.state.save()?;
3606        Ok(())
3607    }
3608
3609    // ---------------------------------------------------------------- vote
3610
3611    async fn vote(&mut self) -> Result<()> {
3612        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3613        // agent files with `magi task add` name the run that paid for it. The
3614        // prompt overlay is cloned alongside it because the waves borrow it
3615        // while `self` is mutably borrowed by the node's own bookkeeping.
3616        let run_id = self.state.id.clone();
3617        let prompts = self.state.config.prompts.clone();
3618        if !self.state.votes.is_empty() {
3619            return Ok(());
3620        }
3621        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3622        if viable.len() == 1 {
3623            return Ok(());
3624        }
3625        self.state.status = RunStatus::Voting;
3626
3627        let language = self.state.config.graph.language.clone();
3628        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3629        let sessions = self.state.config.graph.sessions;
3630        let artifacts = agent::artifacts_dir(&self.state.dir());
3631        let root = self.state.worktree_root();
3632        let base_short = short(&self.state.base_commit);
3633        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3634
3635        let mut jobs = Vec::new();
3636        let mut seats_at = Vec::new();
3637        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3638            if self
3639                .state
3640                .judgements
3641                .get(j)
3642                .is_some_and(|r| r.failed.is_some())
3643            {
3644                continue;
3645            }
3646            let seat_key = format!("judge-{}", j + 1);
3647            let spec = self.occupant(&seat_key, spec);
3648            let seat = self.seat(&seat_key, &spec.id);
3649            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3650            let text = if has_context(&spec, &seat, sessions) {
3651                prompt::final_vote(&viable, &language)
3652            } else {
3653                full.clone()
3654            };
3655            jobs.push(SeatJob {
3656                spec,
3657                seat,
3658                prompt: text,
3659                cwd: root.join(format!("judge-{}", j + 1)),
3660                timeout,
3661                allow_write: false,
3662                sessions,
3663                artifacts: artifacts.clone(),
3664                stem: format!("vote-judge-{}", j + 1),
3665                handover: Some(full),
3666            });
3667            seats_at.push(j);
3668        }
3669
3670        self.state.event(
3671            "vote",
3672            format!(
3673                "collecting {} final votes one by one, privately",
3674                jobs.len()
3675            ),
3676        );
3677        let allowed = viable.clone();
3678        let mut quota_losses = Vec::new();
3679        let cache = self.state.config.cache_dir();
3680        let ctx = WaveCtx {
3681            carry_seats: false,
3682            pauses: Some((&self.pause, &self.interrupt)),
3683            run: &run_id,
3684            node: "vote",
3685            prompts: &prompts,
3686            cache: cache.as_deref(),
3687            round: None,
3688        };
3689        let results = ask_json_wave::<FinalVote>(
3690            jobs,
3691            Arc::clone(&self.sem),
3692            self.state.config.graph.retries,
3693            &self.roles.judge_roster,
3694            &ctx,
3695            &mut quota_losses,
3696            &mut self.state,
3697            &move |v: &FinalVote| match v.label() {
3698                Some(c) if allowed.contains(&c) => Ok(()),
3699                other => bail!("vote {other:?} is not one of {allowed:?}"),
3700            },
3701        )
3702        .await;
3703        if self.state.resource_block.is_some() {
3704            return self.stop_for_resource();
3705        }
3706        self.state.quota.extend(quota_losses);
3707
3708        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3709            let agent_id = seat.agent.clone();
3710            self.state.seats.insert(seat.key.clone(), seat);
3711            let initial = self
3712                .state
3713                .judgements
3714                .get(j)
3715                .and_then(|r| r.ranking.first().copied());
3716            let mut record = VoteRecord {
3717                judge: j + 1,
3718                agent: agent_id,
3719                vote: None,
3720                reason: String::new(),
3721                changed: false,
3722            };
3723            match res {
3724                Ok((v, _)) => {
3725                    record.vote = v.label();
3726                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3727                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3728                    self.state.event(
3729                        "vote",
3730                        format!(
3731                            "judge {} voted {}{}",
3732                            j + 1,
3733                            record.vote.unwrap_or('?'),
3734                            if record.changed { " (changed)" } else { "" }
3735                        ),
3736                    );
3737                }
3738                Err(e) => {
3739                    self.state
3740                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3741                }
3742            }
3743            self.state.votes.push(record);
3744            self.state.save()?;
3745        }
3746        Ok(())
3747    }
3748
3749    // --------------------------------------------------------------- tally
3750
3751    fn tally(&mut self) -> Result<()> {
3752        if self.state.tally.is_some() {
3753            return Ok(());
3754        }
3755        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3756        let tops: Vec<char> = self
3757            .state
3758            .judgements
3759            .iter()
3760            .filter_map(|j| j.ranking.first().copied())
3761            .collect();
3762        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3763
3764        // A judge whose private vote failed still counted once, in the initial
3765        // ranking; using it beats discarding a whole seat.
3766        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3767        let mut cast: Vec<char> = Vec::new();
3768        for (i, j) in self.state.judgements.iter().enumerate() {
3769            let vote = self
3770                .state
3771                .votes
3772                .iter()
3773                .find(|v| v.judge == i + 1)
3774                .and_then(|v| v.vote)
3775                .or_else(|| j.ranking.first().copied());
3776            if let Some(v) = vote {
3777                *first_choice.entry(v).or_insert(0) += 1;
3778                cast.push(v);
3779            }
3780        }
3781
3782        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3783        for j in &self.state.judgements {
3784            let n = j.ranking.len();
3785            for (pos, label) in j.ranking.iter().enumerate() {
3786                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3787            }
3788        }
3789
3790        let best = first_choice.values().copied().max().unwrap_or(0);
3791        let mut leaders: Vec<char> = first_choice
3792            .iter()
3793            .filter(|(_, v)| **v == best)
3794            .map(|(k, _)| *k)
3795            .collect();
3796        let mut tie_break = None;
3797        if leaders.len() > 1 {
3798            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3799            let borda_leaders: Vec<char> = leaders
3800                .iter()
3801                .copied()
3802                .filter(|l| borda[l] == top_borda)
3803                .collect();
3804            tie_break = Some(if borda_leaders.len() == 1 {
3805                format!(
3806                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3807                    leaders.len()
3808                )
3809            } else {
3810                format!(
3811                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3812                    leaders.len()
3813                )
3814            });
3815            leaders = borda_leaders;
3816            leaders.sort_unstable();
3817        }
3818        let winner = *leaders
3819            .first()
3820            .or(viable.first())
3821            .context("no candidate to declare a winner from")?;
3822
3823        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3824        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3825        let deliberated = !self.state.deliberation.is_empty();
3826
3827        // Whose verdict is this? A rate-limited seat is absent even if it
3828        // ranked before the limit hit, so presence is measured against the
3829        // recorded losses, not just "did a ranking ever appear".
3830        let quota_seats: std::collections::BTreeSet<&str> =
3831            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3832        let mut present = 0usize;
3833        for (i, j) in self.state.judgements.iter().enumerate() {
3834            if quota_seats.contains(j.seat.as_str()) {
3835                continue;
3836            }
3837            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3838            let voted = self
3839                .state
3840                .votes
3841                .iter()
3842                .any(|v| v.judge == i + 1 && v.vote.is_some());
3843            if ranked || voted {
3844                present += 1;
3845            }
3846        }
3847        // Strict majority of the configured panel. A bare majority is real
3848        // signal we can act on, while a minority verdict must never stand in
3849        // for a healthy one. A one-candidate run needs no panel at all, and
3850        // `judges` stays `0` rather than the roster size a panel that never
3851        // sat would otherwise be credited with.
3852        let needs_quorum = viable.len() > 1;
3853        let judges_total = if needs_quorum {
3854            self.roles.judges.len()
3855        } else {
3856            0
3857        };
3858        let quorum = if needs_quorum {
3859            judges_total / 2 + 1
3860        } else {
3861            0
3862        };
3863        let met_quorum = !needs_quorum || present >= quorum;
3864        let uncontested = (!needs_quorum).then(|| {
3865            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3866        });
3867
3868        self.state.event(
3869            "tally",
3870            match &uncontested {
3871                Some(reason) => format!("winner {winner} — {reason}"),
3872                None => format!(
3873                    "winner {winner} — votes {} | initial {} | {} changed | \
3874                     {present}/{judges_total} judges{}",
3875                    first_choice
3876                        .iter()
3877                        .map(|(k, v)| format!("{k}:{v}"))
3878                        .collect::<Vec<_>>()
3879                        .join(" "),
3880                    if unanimous_initial {
3881                        "unanimous"
3882                    } else {
3883                        "split"
3884                    },
3885                    changed_votes,
3886                    if met_quorum {
3887                        String::new()
3888                    } else {
3889                        format!(" — below quorum ({quorum} required)")
3890                    },
3891                ),
3892            },
3893        );
3894        if !met_quorum {
3895            self.state.event(
3896                "stall",
3897                format!(
3898                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3899                     the run stops here, resumable"
3900                ),
3901            );
3902        }
3903        self.state.tally = Some(Tally {
3904            first_choice,
3905            borda,
3906            winner,
3907            rankings: tops.len(),
3908            unanimous_initial,
3909            deliberated,
3910            changed_votes,
3911            unanimous_final,
3912            tie_break,
3913            judges: judges_total,
3914            present,
3915            quorum,
3916            met_quorum,
3917            uncontested,
3918        });
3919        self.state.status = if met_quorum {
3920            RunStatus::Reviewing
3921        } else {
3922            RunStatus::Stalled
3923        };
3924        self.state.save()?;
3925        Ok(())
3926    }
3927
3928    // ------------------------------------------------------------- recover
3929
3930    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3931    /// resumed toward completion once the transient cause clears.
3932    ///
3933    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3934    /// it toward the quorum, which is exactly the set of seats whose absence
3935    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3936    /// not depend on which node happened to hit the limit), or an ordinary
3937    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3938    /// seat is never disturbed.
3939    ///
3940    /// A seat that now answers with a usable ranking is "recovered": its
3941    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3942    /// `tally` counts it present again), and its vote re-collected. A seat that
3943    /// still fails keeps its loss and stays absent.
3944    ///
3945    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3946    /// to review/gate/merge), `false` when it is still below quorum (the run
3947    /// stays `Stalled`, still resumable for a later retry).
3948    #[allow(clippy::too_many_lines)]
3949    async fn recover_stall(&mut self) -> Result<bool> {
3950        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3951        // agent files with `magi task add` name the run that paid for it. The
3952        // prompt overlay is cloned alongside it because the waves borrow it
3953        // while `self` is mutably borrowed by the node's own bookkeeping.
3954        let run_id = self.state.id.clone();
3955        let prompts = self.state.config.prompts.clone();
3956        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3957        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3958        // is what keeps a non-quota collapse — or a quota loss recorded at the
3959        // deliberate node — from being a permanent dead-end on `--resume`.
3960        let quota_seats: BTreeSet<&str> =
3961            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3962        let absent: Vec<String> = self
3963            .state
3964            .judgements
3965            .iter()
3966            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3967            .map(|j| j.seat.clone())
3968            .collect();
3969        if absent.is_empty() {
3970            return Ok(false);
3971        }
3972        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3973        if viable.len() <= 1 {
3974            return Ok(false);
3975        }
3976        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3977        let language = self.state.config.graph.language.clone();
3978        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3979        let sessions = self.state.config.graph.sessions;
3980        let artifacts = agent::artifacts_dir(&self.state.dir());
3981        let root = self.state.worktree_root();
3982        let base_short = short(&self.state.base_commit);
3983        let candidates: Vec<Candidate> = viable.clone();
3984
3985        // Map each absent seat key to its 0-based position in `roles.judges`.
3986        let mut positions: Vec<usize> = absent
3987            .iter()
3988            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3989            .collect();
3990        if positions.is_empty() {
3991            return Ok(false);
3992        }
3993        positions.sort_unstable();
3994        positions.dedup();
3995
3996        // Re-rank the lost seats, one blind prompt each.
3997        let mut judge_jobs = Vec::new();
3998        for &j in &positions {
3999            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4000            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4001            let seat_key = format!("judge-{}", j + 1);
4002            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4003            let seat = self.seat(&seat_key, &spec.id);
4004            judge_jobs.push(SeatJob {
4005                spec,
4006                seat,
4007                prompt: prompt::judge(
4008                    &self.state.instruction,
4009                    &views,
4010                    self.roles.judges.len(),
4011                    &base_short,
4012                    &language,
4013                ),
4014                cwd: root.join(seat_key),
4015                timeout,
4016                allow_write: false,
4017                sessions,
4018                artifacts: artifacts.clone(),
4019                stem: format!("judge-{}-recover", j + 1),
4020                handover: None,
4021            });
4022        }
4023
4024        let labels_for_check = labels.clone();
4025        let mut judge_losses = Vec::new();
4026        let retries = self.state.config.graph.retries;
4027        let cache = self.state.config.cache_dir();
4028        let ctx = WaveCtx {
4029            carry_seats: false,
4030            pauses: Some((&self.pause, &self.interrupt)),
4031            run: &run_id,
4032            node: "judge",
4033            prompts: &prompts,
4034            cache: cache.as_deref(),
4035            round: None,
4036        };
4037        let results = ask_json_wave::<Ranking>(
4038            judge_jobs,
4039            Arc::clone(&self.sem),
4040            retries,
4041            &self.roles.judge_roster,
4042            &ctx,
4043            &mut judge_losses,
4044            &mut self.state,
4045            &move |r: &Ranking| r.validate(&labels_for_check),
4046        )
4047        .await;
4048        if self.state.resource_block.is_some() {
4049            self.stop_for_resource()?;
4050            return Ok(false);
4051        }
4052
4053        // Refresh the judgement of every seat that ranked again.
4054        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4055        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4056            let agent_id = seat.agent.clone();
4057            self.state.seats.insert(seat.key.clone(), seat);
4058            let record = &mut self.state.judgements[j];
4059            match res {
4060                Ok((ranking, out)) => {
4061                    record.agent = agent_id;
4062                    record.ranking = ranking.normalized();
4063                    record.reasons = ranking.reasons;
4064                    record.strengths = ranking.strengths;
4065                    record.confidence = ranking.confidence;
4066                    record.failed = None;
4067                    record.duration_ms = out.duration_ms;
4068                    recovered.insert(j);
4069                    self.state.event(
4070                        "recover",
4071                        format!("judge {} ranked again after the limit", j + 1),
4072                    );
4073                }
4074                Err(e) => {
4075                    self.state
4076                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4077                }
4078            }
4079        }
4080
4081        // Re-ask the votes of the seats that recovered a ranking.
4082        let mut vote_jobs = Vec::new();
4083        let mut vote_pos: Vec<usize> = Vec::new();
4084        for &j in &recovered {
4085            let seat_key = format!("judge-{}", j + 1);
4086            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4087            let seat = self.seat(&seat_key, &spec.id);
4088            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4089            let text = if has_context(&spec, &seat, sessions) {
4090                prompt::final_vote(&labels, &language)
4091            } else {
4092                full.clone()
4093            };
4094            vote_jobs.push(SeatJob {
4095                spec,
4096                seat,
4097                prompt: text,
4098                cwd: root.join(seat_key),
4099                timeout,
4100                allow_write: false,
4101                sessions,
4102                artifacts: artifacts.clone(),
4103                stem: format!("vote-judge-{}-recover", j + 1),
4104                handover: Some(full),
4105            });
4106            vote_pos.push(j);
4107        }
4108        let allowed = labels.clone();
4109        let mut vote_losses = Vec::new();
4110        let vote_retries = self.state.config.graph.retries;
4111        let vote_cache = self.state.config.cache_dir();
4112        let ctx = WaveCtx {
4113            carry_seats: false,
4114            pauses: Some((&self.pause, &self.interrupt)),
4115            run: &run_id,
4116            node: "vote",
4117            prompts: &prompts,
4118            cache: vote_cache.as_deref(),
4119            round: None,
4120        };
4121        let votes = ask_json_wave::<FinalVote>(
4122            vote_jobs,
4123            Arc::clone(&self.sem),
4124            vote_retries,
4125            &self.roles.judge_roster,
4126            &ctx,
4127            &mut vote_losses,
4128            &mut self.state,
4129            &move |v: &FinalVote| match v.label() {
4130                Some(c) if allowed.contains(&c) => Ok(()),
4131                other => bail!("vote {other:?} is not one of {allowed:?}"),
4132            },
4133        )
4134        .await;
4135        if self.state.resource_block.is_some() {
4136            self.stop_for_resource()?;
4137            return Ok(false);
4138        }
4139        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4140            let agent_id = seat.agent.clone();
4141            self.state.seats.insert(seat.key.clone(), seat);
4142            match res {
4143                Ok((v, _)) => {
4144                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4145                        rec.vote = v.label();
4146                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4147                    } else {
4148                        self.state.votes.push(VoteRecord {
4149                            judge: j + 1,
4150                            agent: agent_id,
4151                            vote: v.label(),
4152                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4153                            changed: false,
4154                        });
4155                    }
4156                    self.state.event(
4157                        "recover",
4158                        format!("judge {} voted again after the limit", j + 1),
4159                    );
4160                }
4161                Err(e) => {
4162                    self.state
4163                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4164                }
4165            }
4166        }
4167
4168        // A seat that ranked again is present even if its re-vote failed —
4169        // `tally` falls back to the initial ranking's first choice — so clear
4170        // its quota loss. Seats that still fail keep theirs and stay absent.
4171        let recovered_keys: BTreeSet<String> = recovered
4172            .iter()
4173            .map(|&j| format!("judge-{}", j + 1))
4174            .collect();
4175        self.state
4176            .quota
4177            .retain(|q| !recovered_keys.contains(&q.seat));
4178        // A seat that hit the limit again is a fresh loss, not the old one:
4179        // replace the stale entry so the history stays one-per-seat and the
4180        // daemon can tell this attempt's loss from a previous session's.
4181        for loss in judge_losses.into_iter().chain(vote_losses) {
4182            if recovered_keys.contains(&loss.seat) {
4183                continue;
4184            }
4185            self.state.quota.retain(|q| q.seat != loss.seat);
4186            self.state.quota.push(loss);
4187        }
4188
4189        // Recompute the verdict from the refreshed panel.
4190        self.state.tally = None;
4191        self.tally()?;
4192        Ok(self
4193            .state
4194            .tally
4195            .as_ref()
4196            .map(|t| t.met_quorum)
4197            .unwrap_or(false))
4198    }
4199
4200    // ---------------------------------------------------------------- graft
4201
4202    /// Port the strengths the judges attributed to the losing candidates onto
4203    /// the winner, once, between the tally and the review loop.
4204    ///
4205    /// Best-effort by construction: it returns nothing and fails nothing. A
4206    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4207    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4208    /// the run carries on as if the phase were off. See [`graft`] for the
4209    /// decisions and `AGENTS.md` for the invariants.
4210    ///
4211    /// The one exception is a rollback that could not be confirmed: the
4212    /// rejected commit may still be the winner's tip, so the run is blocked
4213    /// (resumable; the record stays `Running`, which makes a resume retry the
4214    /// restore) instead of reviewing, gating and merging it.
4215    async fn graft(&mut self) -> Result<()> {
4216        let Err(e) = self.graft_inner().await else {
4217            return Ok(());
4218        };
4219        if e.is::<crate::repo_lock::ResourceBlocked>() {
4220            return Err(e);
4221        }
4222        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4223            self.state.status = RunStatus::Blocked;
4224            self.state.event("graft", format!("blocked: {e:#}"));
4225            self.state.save()?;
4226            return Err(e);
4227        }
4228        // An error after the `Running` record was saved may have left the
4229        // fixer's work in the tree: the record must not stay unsettled
4230        // (a later resume would reset past the review fixes), so confirm the
4231        // restore now, or block.
4232        if let Some(rec) = self.state.graft.clone()
4233            && !rec.outcome.settled()
4234        {
4235            if let Some(winner) = self.state.winner().cloned() {
4236                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4237                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4238                        return Err(re);
4239                    }
4240                    self.state.status = RunStatus::Blocked;
4241                    self.state.event("graft", format!("blocked: {re:#}"));
4242                    self.state.save()?;
4243                    return Err(re);
4244                }
4245            }
4246            if let Some(g) = self.state.graft.as_mut() {
4247                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4248            }
4249        }
4250        self.state.event("graft", format!("skipped: {e:#}"));
4251        if let Err(e) = self.state.save() {
4252            tracing::warn!("save after graft: {e:#}");
4253        }
4254        Ok(())
4255    }
4256
4257    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4258    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4259    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4260        let _coordination = self.coordinate(wt).await?;
4261        crate::recovery::restore_repair(wt, pre)
4262            .await
4263            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4264    }
4265
4266    /// A skip is a run event, recorded once however often the node is re-entered.
4267    fn graft_skipped(&mut self, why: &str) {
4268        let message = format!("skipped: {why}");
4269        if !self
4270            .state
4271            .events
4272            .iter()
4273            .any(|e| e.node == "graft" && e.message == message)
4274        {
4275            self.state.event("graft", message);
4276        }
4277    }
4278
4279    async fn graft_inner(&mut self) -> Result<()> {
4280        // A round that was started and never finished (crash, kill): the
4281        // fixer may have left anything in the tree, so put the winner back.
4282        if let Some(rec) = self.state.graft.clone()
4283            && !rec.outcome.settled()
4284        {
4285            // Reviews exist only if the graft ended without its record being
4286            // settled (a swallowed error): the tree has moved on since, and
4287            // a reset to `pre` would discard the review fixes.
4288            if self.state.reviews.is_empty()
4289                && let Some(winner) = self.state.winner().cloned()
4290            {
4291                self.graft_restore(&winner.worktree, &rec.pre).await?;
4292            }
4293            if rec.outcome == GraftOutcome::WaitingForRestore {
4294                self.state.graft = None;
4295                self.state.save()?;
4296            } else {
4297                if let Some(g) = self.state.graft.as_mut() {
4298                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4299                }
4300                self.state.event(
4301                    "graft",
4302                    "rolled back: the previous round was interrupted; the winner is untouched",
4303                );
4304                self.state.save()?;
4305                return Ok(());
4306            }
4307        }
4308        let plan = match graft::plan(&self.state) {
4309            Ok(plan) => plan,
4310            Err(skip) => {
4311                if skip != graft::Skip::AlreadyDone {
4312                    self.graft_skipped(skip.reason());
4313                    self.state.save()?;
4314                }
4315                return Ok(());
4316            }
4317        };
4318        let Some(winner) = self.state.winner().cloned() else {
4319            return Ok(());
4320        };
4321        let wt = winner.worktree.clone();
4322        if !git::is_clean(&wt).await? {
4323            self.graft_skipped("the winner's worktree has uncommitted changes");
4324            self.state.save()?;
4325            return Ok(());
4326        }
4327        let pre = git::rev_parse(&wt, "HEAD").await?;
4328        // Saved before the fixer is called: a stop from here on is repaired
4329        // by the branch above on the next entry.
4330        self.state.graft = Some(GraftRecord {
4331            pre: pre.clone(),
4332            post: None,
4333            lines: None,
4334            limit: None,
4335            agent: None,
4336            outcome: GraftOutcome::Running,
4337            requested: graft::requested(&plan),
4338            adoption: None,
4339            adoption_error: None,
4340        });
4341        self.state.event(
4342            "graft",
4343            format!(
4344                "porting strengths of candidate(s) {} onto {}",
4345                plan.strengths
4346                    .iter()
4347                    .map(|(l, _, _)| l.to_string())
4348                    .collect::<Vec<_>>()
4349                    .join(", "),
4350                plan.winner
4351            ),
4352        );
4353        self.state.save()?;
4354
4355        let fix_prompt = prompt::graft(
4356            &self.state.instruction,
4357            plan.winner,
4358            &plan.strengths,
4359            &self.state.config.graph.language,
4360        );
4361        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4362        let sessions = self.state.config.graph.sessions;
4363        let artifacts = agent::artifacts_dir(&self.state.dir());
4364        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4365        let (_, seat, out) = self
4366            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4367                SeatJob {
4368                    prompt: fix_prompt.clone(),
4369                    spec,
4370                    seat,
4371                    cwd: wt.clone(),
4372                    timeout,
4373                    allow_write: true,
4374                    sessions,
4375                    artifacts: artifacts.clone(),
4376                    stem: "graft".to_owned(),
4377                    handover: None,
4378                }
4379            })
4380            .await;
4381        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4382            // Preserve the need to restore earlier fallback edits without
4383            // confusing an unstarted seat with an interrupted invocation.
4384            self.state.graft.as_mut().expect("graft record").outcome =
4385                GraftOutcome::WaitingForRestore;
4386            return self.stop_for_resource();
4387        }
4388        let mut why: Option<String> = None;
4389        match &out {
4390            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4391                why = Some("the fixer produced nothing usable".to_owned());
4392            }
4393            AgentOutcome::Ok(_) => {}
4394            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4395            AgentOutcome::Quota(o) => {
4396                self.state.quota.push(QuotaLoss {
4397                    seat: seat.key.clone(),
4398                    node: "graft".to_owned(),
4399                    at: Timestamp::now(),
4400                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4401                });
4402                why = Some("rate limited (quota)".to_owned());
4403            }
4404            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4405                why = Some(format!(
4406                    "the fixer failed: {}",
4407                    e.lines()
4408                        .next()
4409                        .unwrap_or("")
4410                        .chars()
4411                        .take(160)
4412                        .collect::<String>()
4413                ));
4414            }
4415        }
4416        let agent_id = seat.agent.clone();
4417        self.state.seats.insert(seat.key.clone(), seat);
4418
4419        let mut post = None;
4420        let mut lines = None;
4421        let _coordination = self.coordinate(&wt).await?;
4422        if why.is_none() {
4423            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4424            if let Ok(r) = &rescued {
4425                self.state.note_withheld("graft", &r.withheld);
4426            }
4427            let head = git::rev_parse(&wt, "HEAD").await?;
4428            // What is measured must be what gets reviewed and gated: a failed
4429            // rescue, or anything still uncommitted (withheld lockfiles
4430            // aside, which are never part of the tree), rejects the graft.
4431            let dirty = match &rescued {
4432                Ok(r) => git::status_porcelain(&wt)
4433                    .await
4434                    .map(|st| {
4435                        st.lines().any(|l| {
4436                            !r.withheld
4437                                .iter()
4438                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4439                        })
4440                    })
4441                    .unwrap_or(true),
4442                Err(_) => true,
4443            };
4444            if dirty {
4445                why = Some("the fixer left changes that could not be committed".to_owned());
4446            } else if head == pre {
4447                why = Some("the fixer changed nothing".to_owned());
4448            } else {
4449                lines = git::changed_lines(&wt, &pre, &head).await;
4450                post = Some(head);
4451            }
4452        }
4453        let (adoption, adoption_error) = match &out {
4454            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4455            _ => (None, None),
4456        };
4457        let requested = graft::requested(&plan);
4458        let outcome = match why {
4459            None => {
4460                self.state.event(
4461                    "graft",
4462                    format!(
4463                        "applied: {}",
4464                        lines.map_or_else(
4465                            || "changed lines unavailable".to_owned(),
4466                            |n| format!("{n} changed lines")
4467                        )
4468                    ),
4469                );
4470                GraftOutcome::Applied
4471            }
4472            Some(why) => {
4473                // Not settled until the restore is confirmed: the record
4474                // stays `Running`, so a resume retries it.
4475                crate::recovery::restore_repair(&wt, &pre)
4476                    .await
4477                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4478                self.state.event(
4479                    "graft",
4480                    format!("rolled back: {why}; the winner is untouched"),
4481                );
4482                GraftOutcome::RolledBack(why)
4483            }
4484        };
4485        self.state.graft = Some(GraftRecord {
4486            pre,
4487            post,
4488            lines,
4489            limit: None,
4490            agent: Some(agent_id),
4491            outcome,
4492            requested,
4493            adoption,
4494            adoption_error,
4495        });
4496        self.state.save()?;
4497        Ok(())
4498    }
4499
4500    // ----------------------------------------------------------------- fold
4501
4502    async fn fold_losers(&mut self) -> Result<()> {
4503        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4504            return Ok(());
4505        };
4506        if !self.state.candidate_preservation.is_empty() {
4507            bail!("candidate preservation incomplete; refusing cleanup");
4508        }
4509        let repo = self.state.repo.clone();
4510        let _coordination = self.coordinate(&repo).await?;
4511        let mut folded = Vec::new();
4512        for i in 0..self.state.candidates.len() {
4513            let c = &self.state.candidates[i];
4514            if c.label == winner || c.folded {
4515                continue;
4516            }
4517            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4518            remove_preserved_worktree(&repo, &wt).await?;
4519            if git::branch_exists(&repo, &branch).await?
4520                && !git::branch_delete(&repo, &branch).await?
4521            {
4522                bail!("candidate branch removal refused");
4523            }
4524            self.state.candidates[i].folded = true;
4525            self.state.save()?;
4526            folded.push(label.to_string());
4527        }
4528        // The judges are finished; their checkouts are pure cost from here.
4529        let root = self.state.worktree_root();
4530        for j in 1..=self.roles.judges.len() {
4531            let wt = root.join(format!("judge-{j}"));
4532            if wt.exists() {
4533                remove_preserved_worktree(&repo, &wt).await?;
4534            }
4535        }
4536        // The design-deliberation stage is finished by the time a tally
4537        // exists — same reasoning as the judges above.
4538        if self.state.config.graph.advise {
4539            for k in 1..=self.state.config.graph.advisors {
4540                let wt = root.join(format!("advisor-{k}"));
4541                if wt.exists() {
4542                    remove_preserved_worktree(&repo, &wt).await?;
4543                }
4544            }
4545        }
4546        if !folded.is_empty() {
4547            self.state
4548                .event("fold", format!("folded candidates {}", folded.join(", ")));
4549            self.state.save()?;
4550        }
4551        Ok(())
4552    }
4553
4554    // ------------------------------------------------------------ base sync
4555
4556    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4557    /// anything verifies it.
4558    ///
4559    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4560    /// read whatever is checked out in the winner's worktree. Left alone that
4561    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4562    /// it when the run *branched* - and a run takes long enough that the base
4563    /// has usually moved by the time it gets here. A gate that ran there
4564    /// answers "green on the commit this run started from", not "green on
4565    /// what is about to land", and the difference showed up three times in
4566    /// one day as a green run whose merge would have reverted a file another
4567    /// pull request had already landed.
4568    ///
4569    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4570    /// `land::Step::Rebase` calls, rather than a second implementation of the
4571    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4572    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4573    /// doc warns about ("two answers to a question nobody notices until a
4574    /// diff is wrong").
4575    ///
4576    /// A conflict is not the end of the road: the standing rebase is handed
4577    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4578    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4579    /// land). Once it finishes, review and the gate run as usual on the
4580    /// rebased tree, which is where a breakage the new base caused is caught
4581    /// by the ordinary gate-fix round. magi resolves nothing itself.
4582    ///
4583    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4584    /// `state.base_sync.attempts`, is how many times the base is *rebased
4585    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4586    /// *conflict* was given to a fixer. When the fixer cannot finish the
4587    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4588    /// what was tried (rounds spent, paths still conflicted) and the branch
4589    /// and worktree stay exactly as they were - untouched, for a person to
4590    /// look at - which is also what makes re-entering this function
4591    /// afterwards a no-op instead of a second attempt at the same wall. A
4592    /// push failure ends the same way.
4593    async fn sync_to_base(&mut self) -> Result<()> {
4594        if self.state.status == RunStatus::AlreadyInBase {
4595            return Ok(());
4596        }
4597        let conflicted = self
4598            .state
4599            .base_sync
4600            .as_ref()
4601            .is_some_and(|s| s.conflict.is_some());
4602        let Some(winner) = self.state.winner().cloned() else {
4603            return Ok(());
4604        };
4605
4606        let repo = self.state.repo.clone();
4607        let remote = self.state.config.merge.remote.clone();
4608        let base_branch = self.state.base_branch.clone();
4609        let tracking = format!("{remote}/{base_branch}");
4610
4611        git::fetch(&repo, &remote, &base_branch).await.ok();
4612        // No network, or the remote never had this branch: the run already
4613        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4614        // otherwise), and one that got this far is not blocked by a fetch
4615        // that fails now. It just has no newer tip to compare against.
4616        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4617            return Ok(());
4618        };
4619
4620        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4621        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4622        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4623
4624        // Before any rebase, and before a recorded conflict is honoured: a
4625        // branch whose change reached the base under other commit ids has
4626        // nothing to rebase and nothing to conflict with, and a run that
4627        // already stopped on that phantom conflict recovers here on resume.
4628        // `behind == 0` with head == tip is a branch the base has since taken
4629        // in whole, whether or not a conflict was ever recorded: the ancestry
4630        // proof must still run (`classify` ignores a head still on the start
4631        // commit).
4632        if (behind > 0 || conflicted || head == tip)
4633            && self
4634                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4635                .await?
4636        {
4637            return Ok(());
4638        }
4639        if conflicted {
4640            return Ok(());
4641        }
4642
4643        if behind == 0 {
4644            // A fixer-finished rebase moves the branch ref before the
4645            // winner's worktree is told (`sync_to_head` below). A run that
4646            // died in between resumes here with `behind == 0` and a tree still
4647            // holding the pre-rebase files, which review and the gate would
4648            // then read. That state is exactly: HEAD moved off the tip the
4649            // rebase started from, yet the tree is still identical to that
4650            // tip. A tree with edits of its own differs from it, so nothing
4651            // is thrown away.
4652            if let Some(from) = self
4653                .state
4654                .rebase_fixes
4655                .iter()
4656                .rev()
4657                .find_map(|r| r.from.clone())
4658                && from != head
4659                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4660                    .await
4661                    .is_ok_and(|o| o.ok())
4662            {
4663                git::sync_to_head(&winner.worktree).await?;
4664            }
4665            // An earlier attempt may have rebased the branch locally and died
4666            // before pushing it (only the fresh-rebase arm below pushes).
4667            // Publish it now, so the plain push at PR time is not refused as
4668            // a non-fast-forward. A run already holding a recorded conflict
4669            // never reaches here; that case is out of scope.
4670            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4671            if let Some(why) = &conflict {
4672                self.state.status = RunStatus::Blocked;
4673                self.state.event("land", why.clone());
4674            }
4675            self.state.base_sync = Some(BaseSync {
4676                tip,
4677                behind: 0,
4678                attempts,
4679                conflict,
4680                already_in: None,
4681            });
4682            self.state.save()?;
4683            return Ok(());
4684        }
4685
4686        if attempts >= BASE_SYNC_ROUNDS {
4687            let why = format!(
4688                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4689                 rebase(s); rebasing again would only race it",
4690                winner.branch
4691            );
4692            self.state.status = RunStatus::Blocked;
4693            self.state.base_sync = Some(BaseSync {
4694                tip,
4695                behind,
4696                attempts,
4697                conflict: Some(why.clone()),
4698                already_in: None,
4699            });
4700            self.state.event("land", why);
4701            self.state.save()?;
4702            return Ok(());
4703        }
4704
4705        self.state.event(
4706            "land",
4707            format!(
4708                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4709                winner.branch
4710            ),
4711        );
4712        self.state.save()?;
4713
4714        // The remote's copy of the branch, read now and only if the fetch
4715        // really succeeded (a stale tracking ref must never pin a lease). It is
4716        // pushed over after a rebase only when it is a commit this branch
4717        // already contains, by ancestry or by patch (an earlier rebase of ours
4718        // that never reached the remote): anything else is somebody else's work.
4719        let branch_tracking = format!("{remote}/{}", winner.branch);
4720        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4721        let remote_tip = if self.state.task_pr_branch().is_none()
4722            && matches!(&fetched_branch, Ok(o) if o.ok())
4723        {
4724            git::rev_parse(&repo, &branch_tracking).await.ok()
4725        } else {
4726            None
4727        };
4728        // A remote tip this branch does not contain is somebody else's work:
4729        // rebasing would leave a local tip that can never be pushed. Stop
4730        // before touching anything and say so.
4731        if let Some(theirs) = &remote_tip
4732            && !git::is_ancestor(&repo, theirs, &head).await
4733            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4734                .await
4735                .is_ok_and(|missing| missing.is_empty())
4736        {
4737            let why = format!(
4738                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4739                 them",
4740                short(theirs),
4741                winner.branch
4742            );
4743            self.state.status = RunStatus::Blocked;
4744            self.state.base_sync = Some(BaseSync {
4745                tip,
4746                behind,
4747                attempts,
4748                conflict: Some(why.clone()),
4749                already_in: None,
4750            });
4751            self.state.event("land", why);
4752            self.state.save()?;
4753            return Ok(());
4754        }
4755
4756        let scratch = self.state.dir().join("base-sync");
4757        let rebased = match crate::rebase::rebase_with_fixer(
4758            &mut self.state,
4759            &scratch,
4760            &winner.branch,
4761            &tracking,
4762        )
4763        .await
4764        {
4765            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4766            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4767            Err(e) => Err(e),
4768        };
4769        let attempts = attempts + 1;
4770        match rebased {
4771            Ok(None) => {
4772                // The branch ref moved, but a worktree that already had it
4773                // checked out (the winner's) was not told; sync its index and
4774                // files before anything reads them.
4775                git::sync_to_head(&winner.worktree).await?;
4776                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4777                let mut conflict = None;
4778                if let Some(pinned) = &remote_tip {
4779                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4780                    match pushed {
4781                        Ok(o) if o.ok() => self.state.event(
4782                            "land",
4783                            format!("pushed rebased {} to {remote}", winner.branch),
4784                        ),
4785                        Ok(o) => {
4786                            conflict = Some(format!(
4787                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4788                                winner.branch,
4789                                short(pinned),
4790                                o.stderr.chars().take(600).collect::<String>()
4791                            ));
4792                        }
4793                        Err(e) => {
4794                            conflict = Some(format!(
4795                                "rebased {} locally but could not push it: {e:#}",
4796                                winner.branch
4797                            ));
4798                        }
4799                    }
4800                }
4801                if let Some(why) = &conflict {
4802                    self.state.status = RunStatus::Blocked;
4803                    self.state.event("land", why.clone());
4804                }
4805                self.state.base_sync = Some(BaseSync {
4806                    tip: tip.clone(),
4807                    behind: 0,
4808                    attempts,
4809                    conflict,
4810                    already_in: None,
4811                });
4812                self.state
4813                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4814            }
4815            Ok(Some(conflict)) => {
4816                let why = format!(
4817                    "{} conflicts with {tracking} and did not rebase: {}",
4818                    winner.branch,
4819                    conflict.chars().take(600).collect::<String>()
4820                );
4821                self.state.status = RunStatus::Blocked;
4822                self.state.base_sync = Some(BaseSync {
4823                    tip,
4824                    behind,
4825                    attempts,
4826                    conflict: Some(why.clone()),
4827                    already_in: None,
4828                });
4829                self.state.event("land", why);
4830            }
4831            // A refused Git write or a busy repository: nothing is wrong with
4832            // the branch. The scratch rebase stays standing and the run
4833            // resumes into it (`base_sync` must not claim a conflict).
4834            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4835                self.state.save()?;
4836                return Ok(());
4837            }
4838            Err(e) => {
4839                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4840                self.state.status = RunStatus::Blocked;
4841                self.state.base_sync = Some(BaseSync {
4842                    tip,
4843                    behind,
4844                    attempts,
4845                    conflict: Some(why.clone()),
4846                    already_in: None,
4847                });
4848                self.state.event("land", why);
4849            }
4850        }
4851        self.state.save()?;
4852        Ok(())
4853    }
4854
4855    /// Push a branch an earlier attempt rebased locally but never published,
4856    /// pinned to the remote tip read right after a successful fetch. Returns
4857    /// the reason when the run must stop; `None` when there was nothing to do
4858    /// (no remote copy, the same tip, or a remote copy this branch already
4859    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4860    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4861        // Task competitions publish only after the PR text gate, to their
4862        // public head; never resurrect a candidate branch on the remote.
4863        if self.state.task_pr_branch().is_some() {
4864            return None;
4865        }
4866        let repo = self.state.repo.clone();
4867        let remote = self.state.config.merge.remote.clone();
4868        let fetched = git::fetch(&repo, &remote, branch).await;
4869        if !matches!(&fetched, Ok(o) if o.ok()) {
4870            return None;
4871        }
4872        let branch_tracking = format!("{remote}/{branch}");
4873        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4874        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4875            return None;
4876        }
4877        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4878            .await
4879            .is_ok_and(|missing| missing.is_empty())
4880        {
4881            return Some(format!(
4882                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4883                 them",
4884                short(&theirs)
4885            ));
4886        }
4887        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4888            Ok(o) if o.ok() => {
4889                self.state
4890                    .event("land", format!("pushed rebased {branch} to {remote}"));
4891                None
4892            }
4893            Ok(o) => Some(format!(
4894                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4895                 someone may have pushed): {}",
4896                short(&theirs),
4897                o.stderr.chars().take(600).collect::<String>()
4898            )),
4899            Err(e) => Some(format!(
4900                "{branch} is rebased locally but could not be pushed: {e:#}"
4901            )),
4902        }
4903    }
4904
4905    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4906    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4907    /// whether it did.
4908    ///
4909    /// Checked only when the base is ahead of the branch. A failing check is
4910    /// "not proven" - the ordinary rebase path then decides - never a reason to
4911    /// stop the run.
4912    ///
4913    /// The remote copy of the branch is held to the same standard as the local
4914    /// one: if it carries a tip this worktree does not, that tip must itself be
4915    /// proven in the base, or nothing is settled (a pull request would
4916    /// otherwise be closed over commits nobody checked). The pull request is
4917    /// closed *before* the terminal status is saved; if that fails for a
4918    /// reason other than a refusal (no network, a `gh` error) the run is left
4919    /// `Blocked` with the reason as its conflict, which a resume retries -
4920    /// the same recovery a phantom conflict gets.
4921    async fn settle_already_in(
4922        &mut self,
4923        branch: &str,
4924        tip: &str,
4925        head: &str,
4926        attempts: usize,
4927        behind: usize,
4928    ) -> Result<bool> {
4929        let repo = self.state.repo.clone();
4930        let remote = self.state.config.merge.remote.clone();
4931        let start = self.state.base_commit.clone();
4932        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4933            Ok(Some(e)) => e,
4934            Ok(None) => return Ok(false),
4935            Err(e) => {
4936                tracing::warn!("already-in-base check for {branch}: {e:#}");
4937                return Ok(false);
4938            }
4939        };
4940        let mut verified = vec![head.to_owned()];
4941        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4942        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4943        if matches!(&fetched, Ok(o) if o.ok())
4944            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4945            && theirs != head
4946        {
4947            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4948                Ok(Some(_)) => verified.push(theirs),
4949                _ => return Ok(false),
4950            }
4951        }
4952        let base_branch = self.state.base_branch.clone();
4953        let message = format!(
4954            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4955             land",
4956            evidence.names(),
4957            evidence.proof.as_str()
4958        );
4959        let closed =
4960            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4961                .await;
4962        match closed {
4963            Ok(Ok(url)) => self
4964                .state
4965                .event("land", format!("closed {url}: superseded on {base_branch}")),
4966            Ok(Err(why)) => self
4967                .state
4968                .event("land", format!("did not close a pull request: {why}")),
4969            Err(e) => {
4970                let why = format!(
4971                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4972                     not be closed ({e:#}); resume to retry"
4973                );
4974                self.state.status = RunStatus::Blocked;
4975                self.state.base_sync = Some(BaseSync {
4976                    tip: tip.to_owned(),
4977                    behind,
4978                    attempts,
4979                    conflict: Some(why.clone()),
4980                    already_in: None,
4981                });
4982                self.state.event("land", why);
4983                self.state.save()?;
4984                return Ok(true);
4985            }
4986        }
4987        self.state.status = RunStatus::AlreadyInBase;
4988        self.state.base_sync = Some(BaseSync {
4989            tip: tip.to_owned(),
4990            behind,
4991            attempts,
4992            conflict: None,
4993            already_in: Some(evidence),
4994        });
4995        self.state.event("land", message);
4996        self.state.save()?;
4997        self.settle_questions();
4998        Ok(true)
4999    }
5000
5001    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5002    /// last landed the winner on, once it has run, else the commit the run
5003    /// branched from.
5004    ///
5005    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5006    /// and `vote` all happen before there is a winner to rebase, so they
5007    /// compare every candidate against the branch point on purpose, and a
5008    /// base that moves after they are already done cannot change an answer
5009    /// they already gave.
5010    fn landing_base(&self) -> String {
5011        self.state
5012            .base_sync
5013            .as_ref()
5014            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5015    }
5016
5017    // ------------------------------------------------------- operator fix
5018
5019    /// Route specific, already-recorded review findings to a fixer for a
5020    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5021    /// entry point.
5022    ///
5023    /// Distinct from `review_loop`'s own fix step in three ways: it never
5024    /// runs a reviewer wave, it never spends review-round budget, and what
5025    /// happened is recorded as an [`OperatorFixRequest`] appended to
5026    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5027    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5028    /// and vote must never be rewritten to look like a manufactured blocking
5029    /// verdict.
5030    ///
5031    /// Only meaningful once review has actually concluded: `Ready` (handed
5032    /// off with findings still open, or simply concluded clean while minor
5033    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5034    /// gate failed). Everything else is refused: a run still in progress
5035    /// should simply be resumed, and a `Merged` run's branch has already
5036    /// landed — reopening *this* run's own record cannot change that, so the
5037    /// answer there is a fresh `magi review <branch>`.
5038    ///
5039    /// A real commit here re-verifies through a fresh, ordinary review-only
5040    /// run on the same branch ([`Self::review`]) rather than reopening this
5041    /// run's own `review_loop`: once any round in this run's history went
5042    /// clean, `review_conclusion` treats that as permanent by design (the
5043    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5044    /// way to force one more genuine reviewer wave out of *this* run without
5045    /// either rewriting history or weakening that guarantee for every other
5046    /// caller. A review-only run costs nothing extra — no implementation, no
5047    /// judging, no vote — and exercises the exact same review → verify →
5048    /// gate → (human) merge path, unmodified.
5049    pub async fn fix_selected(
5050        &mut self,
5051        ids: &[String],
5052        reason: &str,
5053        allow_stale: bool,
5054    ) -> Result<()> {
5055        let reason = reason.trim();
5056        if reason.is_empty() {
5057            bail!("a fix request needs a reason — that is the operator's own record of why");
5058        }
5059        if ids.is_empty() {
5060            bail!("no finding id given");
5061        }
5062        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5063            bail!(
5064                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5065                 has already concluded — can be given a targeted fix. A run still \
5066                 in progress should simply be resumed; a `merged` run's branch has \
5067                 already landed, so its answer is a fresh `magi review <branch>`, \
5068                 not reopening this run's own record",
5069                self.state.id,
5070                self.state.status.as_str()
5071            );
5072        }
5073        let Some(winner) = self.state.winner().cloned() else {
5074            bail!("run {} has no winning candidate to fix", self.state.id);
5075        };
5076        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5077            bail!(
5078                "branch `{}` no longer exists; this run cannot be extended",
5079                winner.branch
5080            );
5081        }
5082        let home = crate::run::home();
5083        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5084            bail!(
5085                "run {} is currently being worked on by another magi process",
5086                self.state.id
5087            );
5088        }
5089        // Held for the rest of this call, including the follow-up review
5090        // below: two `magi fix` invocations against the same run must not
5091        // both reach the worktree manipulation further down, which would
5092        // otherwise race to remove and recreate the same directory — see
5093        // [`FixClaim`]'s own doc.
5094        let _claim = FixClaim::acquire(&self.state.dir())?;
5095
5096        // Resolve every id before spending anything — an unknown id refuses
5097        // the whole request rather than silently dropping it — and dedup
5098        // while keeping the operator's own order.
5099        let mut seen = BTreeSet::new();
5100        let mut findings = Vec::new();
5101        let mut missing = Vec::new();
5102        for id in ids {
5103            if !seen.insert(id.clone()) {
5104                continue;
5105            }
5106            match self.state.finding(id) {
5107                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5108                    id: f.id.clone(),
5109                    severity: f.severity,
5110                    reviewer_vote: rec.vote,
5111                    round: round.round,
5112                    round_head: round.head.clone(),
5113                    reviewer: rec.reviewer,
5114                    agent: rec.agent.clone(),
5115                    file: f.file.clone(),
5116                    line: f.line,
5117                    title: f.title.clone(),
5118                    detail: f.detail.clone(),
5119                    outcome: OperatorFixOutcome::Pending,
5120                }),
5121                None => missing.push(id.clone()),
5122            }
5123        }
5124        if !missing.is_empty() {
5125            bail!(
5126                "unknown finding id(s): {}; nothing was changed",
5127                missing.join(", ")
5128            );
5129        }
5130
5131        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5132        let stale_details: Vec<(String, String)> = findings
5133            .iter()
5134            .filter(|f| f.round_head != head_at_request)
5135            .map(|f| (f.id.clone(), f.round_head.clone()))
5136            .collect();
5137        let stale = !stale_details.is_empty();
5138        if stale && !allow_stale {
5139            bail!(
5140                "the branch has moved since some finding(s) were raised — {} — now \
5141                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5142                stale_details
5143                    .iter()
5144                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5145                    .collect::<Vec<_>>()
5146                    .join(", "),
5147                short(&head_at_request)
5148            );
5149        }
5150
5151        let request = OperatorFixRequest {
5152            requested_at: Timestamp::now(),
5153            reason: reason.to_owned(),
5154            findings,
5155            head_at_request: head_at_request.clone(),
5156            allow_stale,
5157            stale,
5158            fix: None,
5159            result_head: None,
5160            follow_up_review_run: None,
5161        };
5162        self.state.event(
5163            "fix",
5164            format!(
5165                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5166                request.findings.len(),
5167                request
5168                    .findings
5169                    .iter()
5170                    .map(|f| f.id.as_str())
5171                    .collect::<Vec<_>>()
5172                    .join(", "),
5173            ),
5174        );
5175        // Recorded now, before any worktree work or the fixer call itself —
5176        // and re-saved at each checkpoint below: a crash at any point after
5177        // this (mid fixer call, mid follow-up review) must not lose the fact
5178        // that this was requested, for which findings, and why. Everything
5179        // past this point reads and writes through `request_index` rather
5180        // than a local variable, since `request` itself is moved here.
5181        self.state.operator_fixes.push(request);
5182        self.state.save()?;
5183        let request_index = self.state.operator_fixes.len() - 1;
5184
5185        // A fresh, dedicated worktree for this one call, never the winner's
5186        // own worktree in place: that one may already be gone (folded away),
5187        // and reusing it in place would leave the branch checked out there
5188        // when the follow-up review below tries to check it out again. Freed
5189        // immediately after, either way — but only once confirmed clean:
5190        // `worktree_remove` is a `git worktree remove --force`, which would
5191        // otherwise discard uncommitted work left there by the operator or
5192        // another process before this had a chance to even look at it.
5193        let repo = self.state.repo.clone();
5194        let coordination = self.coordinate(&repo).await?;
5195        if winner.worktree.exists() {
5196            // Lockfiles a rescue commit withheld stay untracked on purpose and
5197            // are already recorded; they are not the operator's work to protect.
5198            let dirty = git::git(
5199                &winner.worktree,
5200                &["status", "--porcelain", "--untracked-files=all"],
5201            )
5202            .await?;
5203            let only_withheld = dirty.lines().all(|l| {
5204                l.strip_prefix("?? ")
5205                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5206            });
5207            if !only_withheld {
5208                bail!(
5209                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5210                     discard them first",
5211                    winner.worktree.display()
5212                );
5213            }
5214            git::worktree_remove(&self.state.repo, &winner.worktree)
5215                .await
5216                .ok();
5217        }
5218        let fix_worktree = self.state.worktree_root().join("operator-fix");
5219        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5220        git::git(
5221            &self.state.repo,
5222            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5223        )
5224        .await
5225        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5226        if !git::is_clean(&fix_worktree).await? {
5227            git::worktree_remove(&self.state.repo, &fix_worktree)
5228                .await
5229                .ok();
5230            bail!(
5231                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5232                winner.branch
5233            );
5234        }
5235        let repo_for_setup = self.state.repo.clone();
5236        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5237            Ok(()) => {
5238                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5239            }
5240            Err(e) => Err(e),
5241        };
5242        if let Err(e) = setup {
5243            git::worktree_remove(&self.state.repo, &fix_worktree)
5244                .await
5245                .ok();
5246            return Err(e);
5247        }
5248
5249        let run_id = self.state.id.clone();
5250        let prompts = self.state.config.prompts.clone();
5251        let language = self.state.config.graph.language.clone();
5252        let sessions = self.state.config.graph.sessions;
5253        let artifacts = agent::artifacts_dir(&self.state.dir());
5254        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5255            .findings
5256            .iter()
5257            .map(|f| Finding {
5258                id: f.id.clone(),
5259                severity: f.severity,
5260                file: f.file.clone(),
5261                line: f.line,
5262                title: f.title.clone(),
5263                detail: f.detail.clone(),
5264            })
5265            .collect();
5266        let fix_prompt = prompt::operator_fix_in(
5267            &self.state.instruction,
5268            &finding_list,
5269            reason,
5270            &stale_details,
5271            &head_at_request,
5272            &language,
5273            &self.state.config.graph.github_language,
5274        );
5275        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5276        drop(coordination);
5277        let (job, seat, out) = self
5278            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5279                prompt: fix_prompt.clone(),
5280                spec,
5281                seat,
5282                cwd: fix_worktree.clone(),
5283                timeout,
5284                allow_write: true,
5285                sessions,
5286                artifacts: artifacts.clone(),
5287                stem: "operator-fix".to_owned(),
5288                handover: None,
5289            })
5290            .await;
5291        let agent_id = seat.agent.clone();
5292
5293        let mut fix = FixRecord {
5294            agent: agent_id,
5295            addressed: Vec::new(),
5296            rejected: Vec::new(),
5297            notes: String::new(),
5298            committed: false,
5299            failed: None,
5300            duration_ms: 0,
5301            continuation: None,
5302        };
5303        let mut final_seat = seat.clone();
5304        match out {
5305            AgentOutcome::Ok(o) => {
5306                fix.duration_ms = o.duration_ms;
5307                let parsed = verdict::extract_json::<FixReport>(&o.text);
5308                let incomplete_reason = match &parsed {
5309                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5310                        "the reply parsed, but it reported a command whose own CLI \
5311                         never confirmed an exit status"
5312                            .to_owned(),
5313                    ),
5314                    Ok(_) => None,
5315                    Err(e) => Some(e.to_string()),
5316                };
5317                match incomplete_reason {
5318                    None => {
5319                        let report = parsed.expect("checked Ok above");
5320                        fix.addressed = report.addressed;
5321                        fix.rejected = report.rejected;
5322                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5323                    }
5324                    Some(reason) => {
5325                        let (resumed_seat, resolved, failure, cont) = self
5326                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5327                            .await;
5328                        fix.duration_ms += cont.cumulative_wait_ms;
5329                        fix.continuation = Some(cont);
5330                        final_seat = resumed_seat;
5331                        match resolved {
5332                            Some(report) => {
5333                                fix.addressed = report.addressed;
5334                                fix.rejected = report.rejected;
5335                                fix.notes =
5336                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5337                            }
5338                            None => fix.failed = failure,
5339                        }
5340                    }
5341                }
5342            }
5343            AgentOutcome::Dropped(o) => {
5344                fix.duration_ms = o.duration_ms;
5345                let why = o
5346                    .dropped
5347                    .as_ref()
5348                    .map(|d| d.why.as_str())
5349                    .unwrap_or("the CLI ended the stream without delivering its answer");
5350                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5351            }
5352            AgentOutcome::Quota(o) => {
5353                self.state.quota.push(QuotaLoss {
5354                    seat: final_seat.key.clone(),
5355                    node: "fix".to_owned(),
5356                    at: Timestamp::now(),
5357                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5358                });
5359                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5360            }
5361            AgentOutcome::Failed(e) => fix.failed = Some(e),
5362            AgentOutcome::ResourceBlocked(e) => {
5363                fix.failed = Some(format!("the fixer was not started: {e}"));
5364            }
5365        }
5366        if fix.continuation.is_none() {
5367            fix.continuation = Some(ContinuationRecord::not_needed());
5368        }
5369        self.state.seats.insert(final_seat.key.clone(), final_seat);
5370
5371        let rescue_message = format!(
5372            "magi: operator-selected fix ({}) (uncommitted work)",
5373            self.state.operator_fixes[request_index]
5374                .findings
5375                .iter()
5376                .map(|f| f.id.as_str())
5377                .collect::<Vec<_>>()
5378                .join(", ")
5379        );
5380        let repo = self.state.repo.clone();
5381        let coordination = self.coordinate(&repo).await?;
5382        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5383            self.state.note_withheld("fix", &r.withheld);
5384        }
5385        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5386        fix.committed = after != head_at_request;
5387        git::worktree_remove(&self.state.repo, &fix_worktree)
5388            .await
5389            .ok();
5390        drop(coordination);
5391
5392        self.state.event(
5393            "fix",
5394            match &fix.failed {
5395                Some(reason) => format!(
5396                    "operator fix: adoption report was lost ({reason}); {}",
5397                    if fix.committed {
5398                        "committed"
5399                    } else {
5400                        "NO new commit"
5401                    }
5402                ),
5403                None => format!(
5404                    "operator fix: {} addressed, {} rejected, {}",
5405                    fix.addressed.len(),
5406                    fix.rejected.len(),
5407                    if fix.committed {
5408                        "committed"
5409                    } else {
5410                        "NO new commit"
5411                    }
5412                ),
5413            },
5414        );
5415
5416        // Every selected finding gets an outcome — never left `Pending` once
5417        // the fixer's own turn is over. A report that never came back at all
5418        // marks every one of them `Unreported`, not silently "not addressed":
5419        // quota, a dropped stream, or an exhausted continuation are gaps in
5420        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5421        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5422        for f in &mut self.state.operator_fixes[request_index].findings {
5423            f.outcome = if fix.failed.is_some() {
5424                OperatorFixOutcome::Unreported
5425            } else if fix.addressed.contains(&f.id) {
5426                OperatorFixOutcome::Addressed
5427            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5428                OperatorFixOutcome::Rejected { why: r.why.clone() }
5429            } else {
5430                OperatorFixOutcome::Unreported
5431            };
5432        }
5433
5434        let committed = fix.committed;
5435        if committed {
5436            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5437        }
5438        self.state.operator_fixes[request_index].fix = Some(fix);
5439        // Saved again now that the fixer's own outcome is final, on top of
5440        // the save right after the request was first pushed above.
5441        self.state.save()?;
5442
5443        if committed {
5444            self.state.event(
5445                "fix",
5446                format!(
5447                    "operator fix committed {}; opening a follow-up review-only run",
5448                    short(&after)
5449                ),
5450            );
5451            // The operator asked for the fix, and the follow-up serves whatever
5452            // task the run it follows served.
5453            let origin =
5454                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5455            match Self::review(
5456                &self.state.repo,
5457                &winner.branch,
5458                self.state.config.clone(),
5459                origin,
5460            )
5461            .await
5462            {
5463                Ok(mut follow_up) => {
5464                    follow_up.state.event(
5465                        "start",
5466                        format!(
5467                            "requested by an operator fix on run {} for finding(s) {}",
5468                            self.state.id,
5469                            self.state.operator_fixes[request_index]
5470                                .findings
5471                                .iter()
5472                                .map(|f| f.id.as_str())
5473                                .collect::<Vec<_>>()
5474                                .join(", "),
5475                        ),
5476                    );
5477                    follow_up.state.save()?;
5478                    let follow_up_id = follow_up.state.id.clone();
5479                    // The follow-up is a run like any other: it belongs to the
5480                    // task of the run it follows, or to one filed for it.
5481                    let adopted = match crate::direct::adopt(
5482                        &follow_up.state,
5483                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5484                    ) {
5485                        Ok(a) => a,
5486                        Err(e) => {
5487                            // An ownerless run must not spend agent calls; it
5488                            // stays saved, and `magi run --resume` adopts it.
5489                            self.state.event(
5490                                "fix",
5491                                format!(
5492                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5493                                ),
5494                            );
5495                            self.state.save()?;
5496                            return Ok(());
5497                        }
5498                    };
5499                    let executed = follow_up.execute().await;
5500                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5501                    if let Some(a) = adopted {
5502                        a.finish(&follow_up.state, executed);
5503                    }
5504                    if let Some(e) = failure {
5505                        self.state.event(
5506                            "fix",
5507                            format!(
5508                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5509                            ),
5510                        );
5511                    }
5512                    self.state.operator_fixes[request_index].follow_up_review_run =
5513                        Some(follow_up_id);
5514                }
5515                Err(e) => {
5516                    self.state.event(
5517                        "fix",
5518                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5519                    );
5520                }
5521            }
5522            self.state.save()?;
5523        }
5524
5525        Ok(())
5526    }
5527
5528    // --------------------------------------------------------------- review
5529
5530    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5531    /// names, in order, each at most once, moving on only when the call
5532    /// advances (an error, a quota hit or nothing usable - the same decision
5533    /// point `agent::chain_advances` is for the other chained roles).
5534    ///
5535    /// Each agent gets its own seat from `Runner::seat`: the same agent
5536    /// continues its conversation (the winner's own implementer seat when it
5537    /// is the winner's author, now that the competition is over), another
5538    /// takes a fresh one so the full prompt is sent again. A handover is
5539    /// recorded under `node`, which is what makes the fallback stick for the
5540    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5541    /// is returned, so an exhausted chain reads exactly like a single failed
5542    /// fixer: one quota loss, the same wording, the same refund. An earlier
5543    /// attempt's edits are left in the tree and judged, with the final
5544    /// attempt's, by what git says afterwards.
5545    ///
5546    /// Returns the job that produced the outcome, for `continue_fix_report`.
5547    async fn ask_fixer(
5548        &mut self,
5549        winner: &Candidate,
5550        node: &'static str,
5551        round: Option<usize>,
5552        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5553    ) -> (SeatJob, SeatState, AgentOutcome) {
5554        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5555        self.ask_fixer_among(attempts, node, round, None, build)
5556            .await
5557    }
5558
5559    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5560    /// worktree and a commit: before the next agent is asked after one that
5561    /// did not satisfy the call, the worktree is put back on that commit, so
5562    /// a failed agent's partial edits are not carried into the next one.
5563    async fn ask_fixer_among(
5564        &mut self,
5565        attempts: Vec<(AgentSpec, String)>,
5566        node: &'static str,
5567        round: Option<usize>,
5568        restore: Option<(&Path, &str)>,
5569        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5570    ) -> (SeatJob, SeatState, AgentOutcome) {
5571        let run_id = self.state.id.clone();
5572        let prompts = self.state.config.prompts.clone();
5573        let cache = self.state.config.cache_dir();
5574        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5575        let mut last = None;
5576        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5577            let seat = self.seat(&key, &spec.id);
5578            let mut job = build(spec.clone(), seat);
5579            if i > 0 {
5580                job.stem = format!("{}-{}", job.stem, spec.id);
5581            }
5582            let ctx = WaveCtx {
5583                carry_seats: false,
5584                pauses: Some((&self.pause, &self.interrupt)),
5585                run: &run_id,
5586                node,
5587                prompts: &prompts,
5588                cache: cache.as_deref(),
5589                round,
5590            };
5591            let (seat, out) =
5592                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5593            // A resource block never advances the chain: the chosen agent
5594            // was not asked, so it must be the one asked again on resume.
5595            let advances = match &out {
5596                AgentOutcome::Ok(o) => agent::output_advances(o),
5597                AgentOutcome::ResourceBlocked(_) => false,
5598                _ => true,
5599            };
5600            if let Some(next) = ids.get(i + 1)
5601                && advances
5602            {
5603                self.state.seats.insert(seat.key.clone(), seat.clone());
5604                let class =
5605                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5606                let reason = match &out {
5607                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5608                    other => fail_reason(other),
5609                };
5610                // A failed agent's edits must not reach the next one, and a
5611                // restore that did not work ends the chain: the call fails
5612                // and the caller's own (confirmed) restore decides.
5613                if let Some((wt, rev)) = restore
5614                    && let Err(e) = self.graft_restore(wt, rev).await
5615                {
5616                    let why =
5617                        format!("could not restore the worktree before the next fixer: {e:#}");
5618                    tracing::warn!("{why}");
5619                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5620                        AgentOutcome::ResourceBlocked(why)
5621                    } else {
5622                        AgentOutcome::Failed(why)
5623                    };
5624                    last = Some((job, seat, out));
5625                    break;
5626                }
5627                record_handover(
5628                    &mut self.state,
5629                    node,
5630                    &seat.key,
5631                    &spec.id,
5632                    next,
5633                    &class,
5634                    &reason,
5635                );
5636                continue;
5637            }
5638            last = Some((job, seat, out));
5639            break;
5640        }
5641        last.expect("the fixer chain always has an entry")
5642    }
5643
5644    /// The one repair a verification-only failure earns (see
5645    /// [`crate::recovery`]), before the first review round of a run that took
5646    /// a branch over.
5647    ///
5648    /// Saved as started *before* the fixer is called, so a resume never
5649    /// repairs twice; a start found on re-entry is rolled back to the commit
5650    /// it began from. A failed or unusable fixer leaves the tree as it was and
5651    /// the review goes on over it, so the first round's own verification
5652    /// reports the failure again. The failure is never carried into a round as
5653    /// a result: verification is re-run on whatever commit this leaves.
5654    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5655        use crate::recovery::{Mode, Repair};
5656        if !self.state.reviews.is_empty() {
5657            return Ok(());
5658        }
5659        let Some(mut rec) = self.state.recovery.clone() else {
5660            return Ok(());
5661        };
5662        if rec.mode != Mode::VerifyRepair {
5663            return Ok(());
5664        }
5665        let wt = winner.worktree.clone();
5666        let mut head = git::rev_parse(&wt, "HEAD").await?;
5667        match rec.repair.clone() {
5668            Repair::Done { .. } => return Ok(()),
5669            Repair::WaitingForRestore { before } => {
5670                self.graft_restore(&wt, &before).await?;
5671                head = before;
5672                rec.repair = Repair::NotStarted;
5673                self.state.recovery = Some(rec.clone());
5674                self.state.save()?;
5675            }
5676            Repair::Started { before } => {
5677                // Interrupted: the unfinished edits are not trusted.
5678                // An unconfirmed rollback blocks the run with `Started` kept,
5679                // so a resume retries it; the tree is never reviewed as is.
5680                if let Err(e) = self.graft_restore(&wt, &before).await {
5681                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5682                        return Err(e);
5683                    }
5684                    return self
5685                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5686                        .await;
5687                }
5688                rec.repair = Repair::Done {
5689                    before: before.clone(),
5690                    after: before,
5691                    committed: false,
5692                    failed: Some("interrupted; rolled back".to_owned()),
5693                };
5694                self.state
5695                    .event("recovery", "the repair was interrupted and rolled back");
5696                self.state.recovery = Some(rec);
5697                self.state.save()?;
5698                return Ok(());
5699            }
5700            Repair::NotStarted => {}
5701        }
5702        if crate::recovery::recheck(&mut rec, &head) {
5703            self.state.event("recovery", rec.summary());
5704            self.state.recovery = Some(rec);
5705            self.state.save()?;
5706            return Ok(());
5707        }
5708        rec.repair = Repair::Started {
5709            before: head.clone(),
5710        };
5711        self.state.recovery = Some(rec.clone());
5712        self.state.save()?;
5713
5714        let language = self.state.config.graph.language.clone();
5715        let sessions = self.state.config.graph.sessions;
5716        let artifacts = agent::artifacts_dir(&self.state.dir());
5717        let fix_prompt = prompt::recovery_fix(
5718            &self.state.instruction,
5719            &rec,
5720            self.state.config.graph.review_rounds,
5721            &language,
5722        );
5723        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5724        self.state.event(
5725            "recovery",
5726            format!(
5727                "repair: the fixer starts from the failed verification on {}",
5728                short(&head)
5729            ),
5730        );
5731        let (_job, seat, out) = self
5732            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5733                prompt: fix_prompt.clone(),
5734                spec,
5735                seat,
5736                cwd: wt.clone(),
5737                timeout,
5738                allow_write: true,
5739                sessions,
5740                artifacts: artifacts.clone(),
5741                stem: "fix-recovery".to_owned(),
5742                handover: None,
5743            })
5744            .await;
5745        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5746            rec.repair = Repair::WaitingForRestore { before: head };
5747            self.state.recovery = Some(rec);
5748            return self.stop_for_resource();
5749        }
5750        let failed = match &out {
5751            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5752            AgentOutcome::Quota(o) => {
5753                self.state.quota.push(QuotaLoss {
5754                    seat: seat.key.clone(),
5755                    node: "fix".to_owned(),
5756                    at: Timestamp::now(),
5757                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5758                });
5759                Some("rate limited (quota); the fixer could not run".to_owned())
5760            }
5761            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5762            AgentOutcome::Failed(e) => Some(e.clone()),
5763        };
5764        self.state.seats.insert(seat.key.clone(), seat);
5765        let coordination = self.coordinate(&wt).await?;
5766        let failed = match failed {
5767            // The agent's partial edits are not kept.
5768            Some(why) => {
5769                if let Err(e) = git::restore_to(&wt, &head).await {
5770                    return self
5771                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5772                        .await;
5773                }
5774                Some(why)
5775            }
5776            None => {
5777                if let Ok(r) = git::rescue_commit(
5778                    &wt,
5779                    "magi: recovery repair of the failed verification (uncommitted work)",
5780                )
5781                .await
5782                {
5783                    self.state.note_withheld("fix", &r.withheld);
5784                }
5785                None
5786            }
5787        };
5788        let after = git::rev_parse(&wt, "HEAD").await?;
5789        drop(coordination);
5790        self.state.event(
5791            "recovery",
5792            match &failed {
5793                Some(why) => format!("repair: nothing kept ({why})"),
5794                None if after != head => {
5795                    format!("repair: committed {}", short(&after))
5796                }
5797                None => "repair: the fixer made no commit".to_owned(),
5798            },
5799        );
5800        rec.repair = Repair::Done {
5801            before: head.clone(),
5802            committed: after != head,
5803            after,
5804            failed,
5805        };
5806        self.state.recovery = Some(rec);
5807        self.state.save()?;
5808        Ok(())
5809    }
5810
5811    /// A repair whose rollback could not be confirmed: the run is blocked and
5812    /// resumable, and the unconfirmed state is never reviewed.
5813    async fn stop_recovery(
5814        &mut self,
5815        mut rec: crate::recovery::RecoveryRecord,
5816        head: &str,
5817        why: String,
5818    ) -> Result<()> {
5819        // `Started` stays recorded: a resume retries the restore.
5820        rec.repair = crate::recovery::Repair::Started {
5821            before: head.to_owned(),
5822        };
5823        self.state.recovery = Some(rec);
5824        self.state
5825            .event("recovery", format!("repair could not be undone: {why}"));
5826        self.state.status = RunStatus::Blocked;
5827        self.state.save()?;
5828        bail!("recovery repair could not be rolled back: {why}")
5829    }
5830
5831    async fn review_loop(&mut self) -> Result<()> {
5832        // A base that would not rebase is a person's decision, not a review
5833        // round: nothing here would change the answer, and reviewers and a
5834        // fixer would be spending real budget on a tree that cannot land
5835        // regardless of what they find.
5836        if self
5837            .state
5838            .base_sync
5839            .as_ref()
5840            .is_some_and(|s| s.conflict.is_some())
5841        {
5842            return Ok(());
5843        }
5844        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5845        // agent files with `magi task add` name the run that paid for it. The
5846        // prompt overlay is cloned alongside it because the waves borrow it
5847        // while `self` is mutably borrowed by the node's own bookkeeping.
5848        let run_id = self.state.id.clone();
5849        let prompts = self.state.config.prompts.clone();
5850        let Some(winner) = self.state.winner().cloned() else {
5851            return Ok(());
5852        };
5853        let max_rounds = self.state.config.graph.review_rounds;
5854        // A clean round, an exhausted round budget, or a stalled tree (see
5855        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5856        // they are recorded — recomputed here, not read off `status`, so a
5857        // reentry into a run that already stopped restates the identical
5858        // verdict instead of silently handing back whatever an earlier node
5859        // in this same walk clobbered `status` to (a solo-candidate
5860        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5861        // below runs an empty range once the budget is spent, and would
5862        // otherwise fall through without touching `status` at all.
5863        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5864            // A reentry after a crash between the last round's save and
5865            // `stop_reviewing` reaches the hand-off here, not there.
5866            if status == RunStatus::Gating {
5867                self.record_contested_handoff();
5868            }
5869            self.state.status = status;
5870            self.state.save()?;
5871            return Ok(());
5872        }
5873        self.state.status = RunStatus::Reviewing;
5874        // A last recorded round whose own verification never resolved
5875        // (`ResourceBlocked` — the shared build cache, not the patch) is
5876        // never a concluded round, whatever the round budget says: starting
5877        // a fresh round on top of it would spend a whole new reviewer wave
5878        // re-reading an unchanged patch instead of just retrying the one
5879        // check that actually needs it, and once the budget is spent the
5880        // loop below has nothing left to do at all (its range is empty).
5881        // Retry that check directly instead, exactly the same retry
5882        // `stop_reviewing` already does for its own catch-up case.
5883        if self
5884            .state
5885            .reviews
5886            .last()
5887            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5888        {
5889            let shell = self.state.config.shell();
5890            return self
5891                .stop_reviewing(
5892                    "the last round's own verification never resolved",
5893                    &shell,
5894                    &winner.worktree,
5895                )
5896                .await;
5897        }
5898
5899        self.recovery_repair(&winner).await?;
5900        if self.resource_halt()? {
5901            return Ok(());
5902        }
5903
5904        let repo = self.state.repo.clone();
5905        let root = self.state.worktree_root();
5906        let language = self.state.config.graph.language.clone();
5907        let sessions = self.state.config.graph.sessions;
5908        let artifacts = self.merge_repair_artifacts();
5909        let base = self.landing_base();
5910        let base_short = short(&base);
5911        let reviewers = self.roles.reviewers.clone();
5912        let shell = self.state.config.shell();
5913
5914        for round in (self.state.reviews.len() + 1)..=max_rounds {
5915            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5916            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5917            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5918            // The prior round's own record, already persisted — never a
5919            // hand-carried variable of just its failing output: that is
5920            // exactly what let a round's e2e result drift out of sync with
5921            // which commit it was actually about (see `SCHEMA`'s doc for
5922            // schema 8). Judged against `head`, the commit reviewers are
5923            // about to look at now, so the summary always reads as "an
5924            // earlier head" here — this round's own patch has not been
5925            // checked yet.
5926            let prev_verification = self
5927                .state
5928                .reviews
5929                .last()
5930                .and_then(|r| r.verification_summary(&head));
5931            // Only the first round of a run that took a branch over: later
5932            // rounds have this run's own history.
5933            let prior_section = match (&self.state.recovery, round) {
5934                (Some(rec), 1) if self.state.reviews.is_empty() => {
5935                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5936                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5937                            Some(crate::recovery::head_of(
5938                                &git::diff(&winner.worktree, reviewed, "HEAD")
5939                                    .await
5940                                    .unwrap_or_default(),
5941                                crate::recovery::SINCE_DIFF_MAX,
5942                            ))
5943                        }
5944                        _ => None,
5945                    };
5946                    Some(prompt::recovery_review(rec, since.as_deref()))
5947                }
5948                _ => None,
5949            };
5950
5951            // Each reviewer gets its own detached checkout of exactly this
5952            // commit: nobody can perturb the winner's tree, and the fixer can
5953            // keep working without racing a reviewer.
5954            let mut jobs = Vec::new();
5955            let coordination = self.coordinate(&repo).await?;
5956            for (r, spec) in reviewers.iter().cloned().enumerate() {
5957                let wt = root.join(format!("review-{}", r + 1));
5958                // `clean -fdx` in a reset wipes setup's products, so every
5959                // round sets the seat up again.
5960                if wt.exists() {
5961                    git::reset_detached(&wt, &head).await?;
5962                } else {
5963                    git::worktree_add_detached(&repo, &wt, &head).await?;
5964                }
5965                self.setup_seat_worktree(&repo, &wt, false).await?;
5966                let seat_key = format!("review-{}", r + 1);
5967                // The seat starts the round on whoever answered it last, not
5968                // on the agent the spec names, so a failure is not re-paid.
5969                let spec = pick_start_spec(
5970                    &self.roles.reviewer_roster,
5971                    spec,
5972                    self.state.seat_history.get(&seat_key),
5973                );
5974                let seat = self.seat(&seat_key, &spec.id);
5975                jobs.push(SeatJob {
5976                    prompt: prompt::review(&prompt::ReviewCtx {
5977                        instruction: &self.state.instruction,
5978                        branch: &winner.branch,
5979                        base_short: &base_short,
5980                        stat: &stat,
5981                        patch: &patch,
5982                        verification: prev_verification.as_ref(),
5983                        prior: prior_section.as_deref(),
5984                        reviewers: reviewers.len(),
5985                        round,
5986                        rounds: max_rounds,
5987                        // A review-only run has no rankings, so nothing
5988                        // competed for this patch and the reviewer is told so.
5989                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5990                        lens: Lens::for_seat(r),
5991                        language: &language,
5992                        github_language: &self.state.config.graph.github_language,
5993                    }),
5994                    spec,
5995                    seat,
5996                    cwd: wt,
5997                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5998                    allow_write: false,
5999                    sessions,
6000                    artifacts: artifacts.clone(),
6001                    stem: format!("review-{round}-{}", r + 1),
6002                    handover: None,
6003                });
6004            }
6005
6006            drop(coordination);
6007            self.state.event(
6008                "review",
6009                format!(
6010                    "round {round}: {} reviewers on {}",
6011                    jobs.len(),
6012                    short(&head)
6013                ),
6014            );
6015            let mut quota_losses = Vec::new();
6016            let review_retries = self.state.config.graph.retries;
6017            let review_cache = self.state.config.cache_dir();
6018            let ctx = WaveCtx {
6019                carry_seats: true,
6020                pauses: Some((&self.pause, &self.interrupt)),
6021                run: &run_id,
6022                node: "review",
6023                prompts: &prompts,
6024                cache: review_cache.as_deref(),
6025                round: Some(round),
6026            };
6027            let results = ask_json_wave::<Review>(
6028                jobs,
6029                Arc::clone(&self.sem),
6030                review_retries,
6031                &self.roles.reviewer_roster,
6032                &ctx,
6033                &mut quota_losses,
6034                &mut self.state,
6035                &|_: &Review| Ok(()),
6036            )
6037            .await;
6038            if self.state.resource_block.is_some() {
6039                return self.stop_for_resource();
6040            }
6041            // Counted before the move below: how many of *this* round's
6042            // reviewer seats were lost to their own rate limit, as opposed to
6043            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6044            let round_quota_missing = quota_losses.len();
6045            self.state.quota.extend(quota_losses);
6046
6047            let mut records = Vec::new();
6048            let mut all_findings = Vec::new();
6049            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6050                let agent_id = seat.agent.clone();
6051                self.state.seats.insert(seat.key.clone(), seat);
6052                let mut record = ReviewRecord {
6053                    reviewer: r + 1,
6054                    agent: agent_id,
6055                    summary: String::new(),
6056                    findings: Vec::new(),
6057                    vote: None,
6058                    failed: None,
6059                    duration_ms: 0,
6060                    // Set for both outcomes: `failed: Some(_)` with
6061                    // `attempts > 0` is a seat every retry still lost, not a
6062                    // recovered one — only `failed: None` with `attempts > 0`
6063                    // reads as "answered after a nudge" (see this field's own
6064                    // doc).
6065                    attempts,
6066                };
6067                match res {
6068                    Ok((review, out)) => {
6069                        // Sanitized here, at the point every other piece of
6070                        // agent prose in this file is (candidate summaries,
6071                        // deliberation turns, vote reasons): a reviewer's own
6072                        // words are the one thing about it that could name
6073                        // it, and reconsideration below broadcasts this same
6074                        // summary and these same findings to every other
6075                        // seat on the panel.
6076                        record.summary =
6077                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6078                        record.vote = Some(review.vote);
6079                        record.duration_ms = out.duration_ms;
6080                        for (n, mut f) in review.findings.into_iter().enumerate() {
6081                            // ids are magi's, never the agent's: the fixer's
6082                            // adoption report is keyed by them.
6083                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6084                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6085                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6086                            // `file` is agent-supplied prose too, never
6087                            // checked against the real tree — the same
6088                            // exposure `title`/`detail` above have, just in
6089                            // a field easy to forget because it looks like a
6090                            // path rather than free text.
6091                            f.file = f
6092                                .file
6093                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6094                            all_findings.push(f.clone());
6095                            record.findings.push(f);
6096                        }
6097                        self.state.event(
6098                            "review",
6099                            format!(
6100                                "round {round}: reviewer {} voted {} with {} finding(s)",
6101                                r + 1,
6102                                review.vote.label(),
6103                                record.findings.len()
6104                            ),
6105                        );
6106                    }
6107                    Err(e) => {
6108                        record.failed = Some(e.to_string());
6109                        self.state.event(
6110                            "review",
6111                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6112                        );
6113                    }
6114                }
6115                records.push(record);
6116            }
6117
6118            // Tally the round's votes and, if they split, spend the one
6119            // round of reconsideration the split -> deliberate -> revote
6120            // shape `judge`/`vote` use for the panel, sized down to what a
6121            // read-only review round can afford: one round, and a revote
6122            // rather than an argument, because the panel already wrote its
6123            // reasoning down as findings the first time around.
6124            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6125            let vote_split =
6126                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6127            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6128            if vote_split {
6129                self.state.event(
6130                    "review",
6131                    format!(
6132                        "round {round}: votes split ({}) — one round of reconsideration",
6133                        initial_votes
6134                            .iter()
6135                            .map(|v| v.label())
6136                            .collect::<Vec<_>>()
6137                            .join(", ")
6138                    ),
6139                );
6140                // Seats read every seat's findings and votes, still numbered
6141                // and never named — the same anonymity `review` itself keeps.
6142                let panel: Vec<ReviewSeatReport<'_>> = records
6143                    .iter()
6144                    .filter_map(|r| {
6145                        r.vote.map(|vote| ReviewSeatReport {
6146                            reviewer: r.reviewer,
6147                            vote,
6148                            summary: &r.summary,
6149                            findings: &r.findings,
6150                        })
6151                    })
6152                    .collect();
6153
6154                let mut jobs = Vec::new();
6155                let mut seats_at = Vec::new();
6156                for (r, spec) in reviewers.iter().cloned().enumerate() {
6157                    // A seat with no initial vote has nothing to reconsider
6158                    // from and stays absent, the same as it stayed absent
6159                    // from `panel` above.
6160                    if records[r].vote.is_none() {
6161                        continue;
6162                    }
6163                    let wt = root.join(format!("review-{}", r + 1));
6164                    let seat_key = format!("review-{}", r + 1);
6165                    let spec = self.occupant(&seat_key, spec);
6166                    let seat = self.seat(&seat_key, &spec.id);
6167                    // A seat with no live session has already forgotten the
6168                    // initial review's prompt — restate the patch it is
6169                    // voting on, the same as `deliberate`/`vote` do for a
6170                    // judge in the same position.
6171                    // The panel already carries this seat's own review and
6172                    // vote, so restating the patch makes the prompt whole for
6173                    // a seat handed to another agent.
6174                    let build = |with_patch: bool| {
6175                        prompt::review_reconsider(&ReviewReconsiderCtx {
6176                            instruction: &self.state.instruction,
6177                            reviewer: r + 1,
6178                            lens: Lens::for_seat(r),
6179                            panel: &panel,
6180                            patch: with_patch.then_some(ReviewPatch {
6181                                branch: &winner.branch,
6182                                base_short: &base_short,
6183                                stat: &stat,
6184                                patch: &patch,
6185                            }),
6186                            round,
6187                            rounds: max_rounds,
6188                            language: &language,
6189                        })
6190                    };
6191                    let full = build(true);
6192                    let prompt = if has_context(&spec, &seat, sessions) {
6193                        build(false)
6194                    } else {
6195                        full.clone()
6196                    };
6197                    jobs.push(SeatJob {
6198                        prompt,
6199                        spec,
6200                        seat,
6201                        cwd: wt,
6202                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6203                        allow_write: false,
6204                        sessions,
6205                        artifacts: artifacts.clone(),
6206                        stem: format!("review-{round}-reconsider-{}", r + 1),
6207                        handover: Some(full),
6208                    });
6209                    seats_at.push(r);
6210                }
6211
6212                let mut recon_quota_losses = Vec::new();
6213                let recon_cache = self.state.config.cache_dir();
6214                let recon_ctx = WaveCtx {
6215                    carry_seats: true,
6216                    pauses: Some((&self.pause, &self.interrupt)),
6217                    run: &run_id,
6218                    node: "review",
6219                    prompts: &prompts,
6220                    cache: recon_cache.as_deref(),
6221                    round: Some(round),
6222                };
6223                let recon_results = ask_json_wave::<ReviewRevote>(
6224                    jobs,
6225                    Arc::clone(&self.sem),
6226                    review_retries,
6227                    &self.roles.reviewer_roster,
6228                    &recon_ctx,
6229                    &mut recon_quota_losses,
6230                    &mut self.state,
6231                    &|_: &ReviewRevote| Ok(()),
6232                )
6233                .await;
6234                if self.state.resource_block.is_some() {
6235                    return self.stop_for_resource();
6236                }
6237                self.state.quota.extend(recon_quota_losses);
6238
6239                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6240                    let agent_id = seat.agent.clone();
6241                    self.state.seats.insert(seat.key.clone(), seat);
6242                    let mut rec = ReviewRevoteRecord {
6243                        reviewer: r + 1,
6244                        agent: agent_id,
6245                        vote: None,
6246                        reason: String::new(),
6247                        failed: None,
6248                    };
6249                    match res {
6250                        Ok((rv, _)) => {
6251                            rec.vote = Some(rv.vote);
6252                            rec.reason =
6253                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6254                            self.state.event(
6255                                "review",
6256                                format!(
6257                                    "round {round}: reviewer {} revoted {}",
6258                                    r + 1,
6259                                    rv.vote.label()
6260                                ),
6261                            );
6262                        }
6263                        Err(e) => {
6264                            rec.failed = Some(e.to_string());
6265                            self.state.event(
6266                                "review",
6267                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6268                            );
6269                        }
6270                    }
6271                    reconsideration.push(rec);
6272                }
6273            } else if initial_votes.len() > 1 {
6274                self.state.event(
6275                    "review",
6276                    format!(
6277                        "round {round}: votes agreed ({}) — no reconsideration",
6278                        initial_votes[0].label()
6279                    ),
6280                );
6281            }
6282
6283            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6284            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6285            // A round that already has a blocking finding and a round left to
6286            // try is going back to the fixer no matter what `verify.e2e`
6287            // says, so running it first only spends the loop's slowest step
6288            // (minutes, for a Rust repo's full test suite) on a head about
6289            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6290            // runs once a round has no blocking findings left (see
6291            // `round_is_clean`, which a deferred — empty — `e2e` can never
6292            // satisfy since `blocking` is nonzero whenever this branch is
6293            // taken), and `stop_reviewing` forces a real run before it will
6294            // ever read a deferred round as green.
6295            let defer_e2e =
6296                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6297            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6298                let reason =
6299                    format!("{blocking} blocking finding(s) already required a fix this round");
6300                self.state.event(
6301                    "verify",
6302                    format!(
6303                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6304                         {}); it will run once a round has none left",
6305                        short(&head)
6306                    ),
6307                );
6308                (Vec::new(), false, true, Some(reason))
6309            } else {
6310                let e2e_commands = self.state.config.verify.e2e.clone();
6311                let cache_dir = self.state.config.cache_dir();
6312                let context = format!("round {round}");
6313                let wait = self.repository_wait();
6314                let (e2e, verify_retried) = with_cache_lease(
6315                    &mut self.state,
6316                    cache_dir.as_deref(),
6317                    "e2e",
6318                    "e2e",
6319                    &winner.worktree,
6320                    &head,
6321                    verify_timeout,
6322                    &context,
6323                    &wait,
6324                    |state, budget| {
6325                        let shell = shell.clone();
6326                        let e2e_commands = e2e_commands.clone();
6327                        let worktree = winner.worktree.clone();
6328                        let context = context.clone();
6329                        async move {
6330                            run_e2e_with_retry(
6331                                state,
6332                                &shell,
6333                                &e2e_commands,
6334                                &worktree,
6335                                budget,
6336                                &context,
6337                            )
6338                            .await
6339                        }
6340                    },
6341                )
6342                .await?;
6343                (e2e, verify_retried, false, None)
6344            };
6345
6346            let expected = records.len();
6347            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6348            let incomplete = answered < expected;
6349            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6350            let policy = self.state.config.graph.incomplete_review;
6351            let clean = round_is_clean(
6352                blocking,
6353                e2e_ok,
6354                answered,
6355                expected,
6356                round_quota_missing,
6357                policy,
6358            );
6359
6360            let mut round_record = ReviewRound {
6361                round,
6362                head: head.clone(),
6363                verified_head: None,
6364                verified_at: None,
6365                reviews: records,
6366                e2e,
6367                verify_retried,
6368                e2e_deferred,
6369                e2e_defer_reason,
6370                fix: None,
6371                blocking,
6372                answered,
6373                expected,
6374                clean,
6375                progressed: false,
6376                vote_split,
6377                reconsideration,
6378                verdict: None,
6379            };
6380            // The final vote per seat is its revote where reconsideration
6381            // ran and answered, its initial vote otherwise — the same
6382            // fallback `tally` uses for a judge whose private vote failed.
6383            round_record.verdict = ReviewVote::worst(
6384                round_record
6385                    .final_votes()
6386                    .into_iter()
6387                    .map(|(_, _, vote)| vote),
6388            );
6389            // Which commit and when magi actually attempted to check —
6390            // known the moment a command was dispatched against `head`,
6391            // whether or not it finished: a resource-blocked attempt still
6392            // targeted a specific commit at a specific time, and leaving
6393            // that unrecorded is exactly what made `verification_summary`
6394            // report a fresh attempt as "commit unknown ... recorded before
6395            // this was tracked", indistinguishable from a genuinely old,
6396            // untracked record. Only a deferred or unconfigured round never
6397            // ran at all and has nothing to record — see
6398            // `ReviewRound::verified_head`'s own doc.
6399            if !matches!(
6400                round_record.e2e_status(),
6401                E2eStatus::Deferred | E2eStatus::NotConfigured
6402            ) {
6403                round_record.verified_head = Some(head.clone());
6404                round_record.verified_at = Some(Timestamp::now());
6405            }
6406            let this_round_verification = round_record.verification_summary(&head);
6407
6408            if incomplete {
6409                let missing: Vec<String> = round_record
6410                    .reviews
6411                    .iter()
6412                    .filter(|r| r.failed.is_some())
6413                    .map(|r| format!("review-{}", r.reviewer))
6414                    .collect();
6415                self.state.event(
6416                    "review",
6417                    format!(
6418                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6419                        missing.join(", ")
6420                    ),
6421                );
6422            }
6423
6424            if clean {
6425                self.state.event(
6426                    "review",
6427                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6428                        format!(
6429                            "round {round}: clean (warn policy, incomplete panel) — no \
6430                             blocking findings from the seats that answered, verification green"
6431                        )
6432                    } else if incomplete {
6433                        format!(
6434                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6435                             quorum) — no blocking findings from the seats that answered, \
6436                             verification green",
6437                            expected - answered
6438                        )
6439                    } else {
6440                        format!("round {round}: clean — no blocking findings, verification green")
6441                    },
6442                );
6443                self.state.reviews.push(round_record);
6444                self.state.status = RunStatus::Gating;
6445                self.state.save()?;
6446                return Ok(());
6447            }
6448
6449            // Nothing was raised and verification passed, but not every seat
6450            // answered and `round_is_clean` still refused to call it clean —
6451            // either a seat is missing for a reason other than its own quota
6452            // (a crash, a timeout, unparsable output — worth another try), or
6453            // every seat that could have answered lost its quota and nobody
6454            // is left to decide on: re-review rather than send the fixer
6455            // after a round with nothing to fix.
6456            if incomplete && blocking == 0 && e2e_ok {
6457                self.state.reviews.push(round_record);
6458                self.state.save()?;
6459                if round == max_rounds {
6460                    self.state.status = RunStatus::Blocked;
6461                    self.state.event(
6462                        "review",
6463                        format!(
6464                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6465                             refusing to call it clean",
6466                            expected - answered
6467                        ),
6468                    );
6469                    return Ok(());
6470                }
6471                continue;
6472            }
6473
6474            // Nothing for the fixer to act on (`blocking == 0`) and the only
6475            // reason this round is not clean is that magi itself never got
6476            // a command to run — the shared build cache, not the patch (see
6477            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6478            // the fixer would invite a change to appease contention that has
6479            // nothing to do with the diff, and would leave this attempt
6480            // sitting in the next round's prompt as if it were about an
6481            // earlier, superseded commit rather than what it actually is:
6482            // the same head, still waiting to be checked. Wait for it the
6483            // same way the final round's own contention is already handled,
6484            // whatever round this happens to be.
6485            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6486                self.state.reviews.push(round_record);
6487                return self
6488                    .stop_reviewing(
6489                        "the round's own verification could not run",
6490                        &shell,
6491                        &winner.worktree,
6492                    )
6493                    .await;
6494            }
6495
6496            if round == max_rounds {
6497                self.state.reviews.push(round_record);
6498                return self
6499                    .stop_reviewing(
6500                        &format!(
6501                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6502                        ),
6503                        &shell,
6504                        &winner.worktree,
6505                    )
6506                    .await;
6507            }
6508
6509            // Fix. The winner's own implementer seat continues its conversation:
6510            // the competition is over, so context is pure benefit now.
6511            let blocking_findings: Vec<_> = all_findings
6512                .iter()
6513                .filter(|f| f.severity.blocks())
6514                .cloned()
6515                .collect();
6516            let fix_prompt = prompt::fix_in(
6517                &self.state.instruction,
6518                &blocking_findings,
6519                this_round_verification.as_ref(),
6520                round,
6521                max_rounds,
6522                &language,
6523                &self.state.config.graph.github_language,
6524            );
6525            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6526            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6527            let (job, seat, out) = self
6528                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6529                    prompt: fix_prompt.clone(),
6530                    spec,
6531                    seat,
6532                    cwd: winner.worktree.clone(),
6533                    timeout,
6534                    allow_write: true,
6535                    sessions,
6536                    artifacts: artifacts.clone(),
6537                    stem: format!("fix-{round}"),
6538                    handover: None,
6539                })
6540                .await;
6541            // Not asked: the round is not recorded (so a resume repeats it),
6542            // no fixer was handed over to, and the seat is left as it was.
6543            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6544                return self.stop_for_resource();
6545            }
6546            let agent_id = seat.agent.clone();
6547
6548            let mut fix = FixRecord {
6549                agent: agent_id,
6550                addressed: Vec::new(),
6551                rejected: Vec::new(),
6552                notes: String::new(),
6553                committed: false,
6554                failed: None,
6555                duration_ms: 0,
6556                continuation: None,
6557            };
6558            let mut continuation = ContinuationRecord::not_needed();
6559            let mut final_seat = seat.clone();
6560            match out {
6561                AgentOutcome::Ok(o) => {
6562                    fix.duration_ms = o.duration_ms;
6563                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6564                    // A parsed report standing next to a command this same
6565                    // reply's own CLI never confirmed the exit status of is
6566                    // not a resolved answer — the identical `CommandEvidence`
6567                    // `state.jobs` renders, read here instead of only on
6568                    // display, per the completion judgment and the shown
6569                    // record needing to agree.
6570                    let incomplete_reason = match &parsed {
6571                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6572                            "the reply parsed, but it reported a command whose own CLI never \
6573                             confirmed an exit status"
6574                                .to_owned(),
6575                        ),
6576                        Ok(_) => None,
6577                        Err(e) => Some(e.to_string()),
6578                    };
6579                    match incomplete_reason {
6580                        None => {
6581                            let report = parsed.expect("checked Ok above");
6582                            fix.addressed = report.addressed;
6583                            fix.rejected = report.rejected;
6584                            fix.notes =
6585                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6586                        }
6587                        Some(reason) => {
6588                            let (resumed_seat, resolved, failure, cont) = self
6589                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6590                                .await;
6591                            fix.duration_ms += cont.cumulative_wait_ms;
6592                            continuation = cont;
6593                            final_seat = resumed_seat;
6594                            match resolved {
6595                                Some(report) => {
6596                                    fix.addressed = report.addressed;
6597                                    fix.rejected = report.rejected;
6598                                    fix.notes = blind::sanitize_prose(
6599                                        &report.notes,
6600                                        &self.state.config.blind,
6601                                    );
6602                                }
6603                                None => fix.failed = failure,
6604                            }
6605                        }
6606                    }
6607                }
6608                // The CLI's raw error JSON is not a fix report to parse.
6609                AgentOutcome::Dropped(o) => {
6610                    fix.duration_ms = o.duration_ms;
6611                    let why = o
6612                        .dropped
6613                        .as_ref()
6614                        .map(|d| d.why.as_str())
6615                        .unwrap_or("the CLI ended the stream without delivering its answer");
6616                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6617                }
6618                AgentOutcome::Quota(o) => {
6619                    self.state.quota.push(QuotaLoss {
6620                        seat: final_seat.key.clone(),
6621                        node: "fix".to_owned(),
6622                        at: Timestamp::now(),
6623                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6624                    });
6625                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6626                }
6627                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6628            }
6629            fix.continuation = Some(continuation);
6630            self.state.seats.insert(final_seat.key.clone(), final_seat);
6631            let coordination = self.coordinate(&winner.worktree).await?;
6632            if let Ok(r) = git::rescue_commit(
6633                &winner.worktree,
6634                &format!("magi: review round {round} fixes (uncommitted work)"),
6635            )
6636            .await
6637            {
6638                self.state.note_withheld("fix", &r.withheld);
6639            }
6640            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6641            fix.committed = after != before;
6642            // Judged by what `git` says moved against base, never by the
6643            // fixer's own `addressed`/`rejected` count — see
6644            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6645            // `patch` snapshot above: swallowing this error would default
6646            // `diff_after` to empty, which almost always differs from a
6647            // non-empty `patch` and reads as "progressed" — exactly backwards
6648            // for a `git` failure the stagnation check cannot see through.
6649            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6650            drop(coordination);
6651            let progressed = diff_after != patch;
6652            let commit_note = if fix.committed {
6653                "committed"
6654            } else {
6655                "NO new commit"
6656            };
6657            let tree_note = if progressed {
6658                "changed vs base"
6659            } else {
6660                "unchanged vs base"
6661            };
6662            self.state.event(
6663                "fix",
6664                match &fix.failed {
6665                    // Distinct on purpose from "0 addressed, 0 rejected": the
6666                    // fixer's own diff still landed (blocking counts do keep
6667                    // falling round over round), only its adoption report did
6668                    // not come back, so this must never read like every
6669                    // finding was reviewed and declined.
6670                    Some(reason) => {
6671                        format!(
6672                            "round {round}: fixer's adoption report was lost ({reason}); \
6673                             {commit_note}, tree {tree_note}"
6674                        )
6675                    }
6676                    None => format!(
6677                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6678                         {tree_note}{}",
6679                        fix.addressed.len(),
6680                        fix.rejected.len(),
6681                        if continuation.outcome == ContinuationOutcome::Resumed {
6682                            format!(
6683                                " (adoption report recovered after {} continuation(s))",
6684                                continuation.attempts
6685                            )
6686                        } else {
6687                            String::new()
6688                        },
6689                    ),
6690                },
6691            );
6692            round_record.fix = Some(fix);
6693            round_record.progressed = progressed;
6694            self.state.reviews.push(round_record);
6695            self.state.save()?;
6696
6697            // The fixer's own report never came back this round, even after
6698            // `continue_fix_report`'s own budget was spent on it — not an
6699            // ordinary "no report" (dropped stream, quota, plain failure),
6700            // which already reads that way and is left to the existing round
6701            // budget. Stopping here, rather than opening another round, is
6702            // what keeps a next reviewer/fixer wave from ever being
6703            // dispatched onto `winner.worktree` while whatever the seat's
6704            // last call may still have running there is unaccounted for: no
6705            // process liveness check exists (and none is being added — see
6706            // AGENTS.md/this task's own scope), so the only way to honour
6707            // "nothing starts before a valid report returns" is to not start
6708            // anything further on this worktree from this run at all.
6709            if matches!(
6710                continuation.outcome,
6711                ContinuationOutcome::Exhausted
6712                    | ContinuationOutcome::QuotaLost
6713                    | ContinuationOutcome::NoSession
6714            ) {
6715                return self
6716                    .stop_reviewing(
6717                        "the fixer's adoption report never came back, even after resuming its \
6718                         own seat; refusing to start another round against the same worktree \
6719                         while that is unresolved",
6720                        &shell,
6721                        &winner.worktree,
6722                    )
6723                    .await;
6724            }
6725
6726            let streak = self
6727                .state
6728                .reviews
6729                .iter()
6730                .rev()
6731                .take_while(|r| !r.progressed)
6732                .count();
6733            if streak >= STAGNANT_LIMIT {
6734                return self
6735                    .stop_reviewing(
6736                        &format!(
6737                            "the tree has not moved against base for {streak} round(s) in a row"
6738                        ),
6739                        &shell,
6740                        &winner.worktree,
6741                    )
6742                    .await;
6743            }
6744        }
6745        Ok(())
6746    }
6747
6748    /// Decide, from the last recorded round's own verification, whether
6749    /// stopping the review loop is a hand-off or a genuine block.
6750    ///
6751    /// Called once the loop has given up trying — the round budget is spent,
6752    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6753    /// findings still open, never while a round is still clean or the
6754    /// incomplete-panel case handled inline above. Gate and e2e are facts
6755    /// about the tree; a lingering review finding is an opinion, and this
6756    /// workload's own `magi stats` puts reviewer precision low enough
6757    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6758    /// must not by itself stand between a green, verified change and the
6759    /// human who decides what to do with it. A red e2e is not an opinion, so
6760    /// that case still blocks, with the failing command and a tail of its
6761    /// output recorded here rather than left in `run.json` for someone to go
6762    /// find.
6763    ///
6764    /// A round that deferred its own e2e (see [`Config::graph`]'s
6765    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6766    /// only because nothing ran, and treating an empty list as a passing one
6767    /// here is exactly the "deferred painted green" bug this function exists
6768    /// to not have. When the last round's own verification never resolved —
6769    /// deferred on purpose, or a real attempt the shared build cache blocked
6770    /// — this makes (or retries) the real run, on the actual worktree this
6771    /// loop is about to stop touching, before deciding anything. A
6772    /// resource-blocked attempt is likewise never read as either green or
6773    /// red: it is evidence about the machine, not the patch (see
6774    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6775    /// blocked cache leaves this call without deciding rather than guessing
6776    /// — the caller retries on a later reentry.
6777    /// Record, once, that the review loop handed off over a blocking finding
6778    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6779    /// `land` asks the owner even with `land_approval` off. Called from every
6780    /// path that concludes `Gating`; a reentry keeps the first record.
6781    fn record_contested_handoff(&mut self) {
6782        if self.state.contested_handoff.is_some() {
6783            return;
6784        }
6785        let Some(contested) = self
6786            .state
6787            .reviews
6788            .last()
6789            .and_then(ReviewRound::contested_handoff)
6790        else {
6791            return;
6792        };
6793        self.state.event(
6794            "review",
6795            format!(
6796                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6797                 wait for the owner's approval",
6798                contested.findings.len(),
6799                contested.rejecters.len()
6800            ),
6801        );
6802        self.state.contested_handoff = Some(contested);
6803    }
6804
6805    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6806        let round_idx = self.state.reviews.len() - 1;
6807        // A deferred round and a resource-blocked one are the same shape
6808        // here: neither has a real result yet, and both get one more
6809        // attempt. Read off `e2e_status` — the single source for this —
6810        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6811        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6812        // still retries instead of being read as a settled result the
6813        // instant it stops being empty.
6814        let needs_catchup_run = matches!(
6815            self.state.reviews[round_idx].e2e_status(),
6816            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6817        );
6818        if needs_catchup_run {
6819            let round = self.state.reviews[round_idx].round;
6820            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6821            let commands = self.state.config.verify.e2e.clone();
6822            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6823            let cache_dir = self.state.config.cache_dir();
6824            let context = format!(
6825                "round {round}: verification unresolved, catching up before the final decision"
6826            );
6827            let wait = self.repository_wait();
6828            let (outcomes, verify_retried) = with_cache_lease(
6829                &mut self.state,
6830                cache_dir.as_deref(),
6831                "e2e",
6832                "e2e",
6833                worktree,
6834                &attempted_head,
6835                timeout,
6836                &context,
6837                &wait,
6838                |state, budget| {
6839                    let shell = shell.to_vec();
6840                    let commands = commands.clone();
6841                    let context = context.clone();
6842                    async move {
6843                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6844                            .await
6845                    }
6846                },
6847            )
6848            .await?;
6849            let last = &mut self.state.reviews[round_idx];
6850            last.e2e = outcomes;
6851            last.verify_retried = verify_retried;
6852            // Always the commit and time this attempt actually targeted,
6853            // whether or not it happens to equal the reviewed `head` and
6854            // whether or not a command finished — see
6855            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6856            // attempt is recorded too, so a later reader sees "attempted
6857            // again at T2" rather than silence.
6858            last.verified_head = Some(attempted_head);
6859            last.verified_at = Some(Timestamp::now());
6860            if verify_inconclusive(&last.e2e) {
6861                // Still not a real result: `e2e_deferred` is left exactly
6862                // as it was, so `needs_catchup_run` above reads
6863                // `ResourceBlocked` (via `e2e_status`, which checks
6864                // `resource_blocked` before `e2e_deferred`) and retries
6865                // again on the next reentry, rather than recording
6866                // contention as a red e2e and blocking the run on it.
6867                self.state.save()?;
6868                return Ok(());
6869            }
6870            last.e2e_deferred = false;
6871        }
6872        let last = &self.state.reviews[round_idx];
6873        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6874
6875        match last.e2e_status() {
6876            E2eStatus::Failed => {
6877                let red: Vec<String> = last
6878                    .e2e
6879                    .iter()
6880                    .filter(|o| !o.ok())
6881                    .map(|o| {
6882                        format!(
6883                            "`{}` -> {:?}\n{}",
6884                            o.command,
6885                            o.code,
6886                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6887                        )
6888                    })
6889                    .collect();
6890                self.state
6891                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6892                self.state.status = RunStatus::Blocked;
6893            }
6894            // `needs_catchup_run` above already retried once this call; if
6895            // it is still blocked, this is magi's own admission it could
6896            // not get a command to run, never a verdict on the patch — the
6897            // run is left exactly where a later reentry can retry again.
6898            E2eStatus::ResourceBlocked => {
6899                self.state.event(
6900                    "review",
6901                    format!(
6902                        "{why}; e2e could not run (shared build cache unavailable); not \
6903                         deciding yet"
6904                    ),
6905                );
6906            }
6907            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6908                self.state.event(
6909                    "review",
6910                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6911                );
6912                self.record_contested_handoff();
6913                self.state.status = RunStatus::Gating;
6914            }
6915        }
6916        self.state.save()?;
6917        Ok(())
6918    }
6919
6920    // ----------------------------------------------------------------- gate
6921
6922    async fn gate(&mut self) -> Result<()> {
6923        // Judged by the review record itself, not by `status`: a solo
6924        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6925        // reentry (see `judge`), and trusting it here is exactly how a run
6926        // that exhausted its review budget got gated and merged a second
6927        // time around. `review_conclusion` recomputes the review loop's own
6928        // verdict from the round records themselves — `Gating` for a clean
6929        // round or a hand-off (see `stop_reviewing`), anything else means the
6930        // loop is still going or genuinely blocked.
6931        // A base the winner could not be replayed onto is a decision, not a
6932        // round: there is no landing tree to gate. Read as its own record for
6933        // the same reason the review verdict is.
6934        if self.state.status == RunStatus::Failed
6935            || self
6936                .state
6937                .base_sync
6938                .as_ref()
6939                .is_some_and(|s| s.conflict.is_some())
6940            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6941                != Some(RunStatus::Gating)
6942        {
6943            return Ok(());
6944        }
6945        if self.state.gate_ran {
6946            // `review_loop` derives its conclusion from the clean review
6947            // record on every reentry and therefore puts a completed run back
6948            // in `Gating`. A recorded gate is a stronger, terminal fact:
6949            // retain its original command output (or lack of any, for a repo
6950            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6951            // doc) and restore `Blocked` on a real failure rather than
6952            // pretending the command is still running or running it a second
6953            // time. `gate_ran == false` remains the only shape — unattempted,
6954            // or a resource-blocked retry — that may still need to execute a
6955            // command.
6956            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6957                self.state.status = RunStatus::Blocked;
6958                self.state.save()?;
6959            }
6960            return Ok(());
6961        }
6962        let Some(winner) = self.state.winner().cloned() else {
6963            return Ok(());
6964        };
6965        self.state.status = RunStatus::Gating;
6966        let mut outcomes = self.run_gate(&winner).await?;
6967        loop {
6968            // A resource-blocked outcome means the gate command never actually
6969            // ran - the shared build cache could not be acquired or confirmed
6970            // fresh in time - which is evidence about the machine, not about
6971            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6972            // Recording it as a red gate would mark a run `Blocked` on nothing
6973            // but contention magi has already logged; leaving `self.state.gate`
6974            // empty and `self.state.gate_ran` false instead keeps the shape
6975            // this function already treats as "still needs to run" (see the
6976            // early-return above), so the next call retries the command
6977            // rather than concluding anything.
6978            if verify_inconclusive(&outcomes) {
6979                self.state.save()?;
6980                return Ok(());
6981            }
6982            if outcomes.iter().all(CommandOutcome::ok) {
6983                break;
6984            }
6985            match self.gate_fix_round(&winner, &outcomes).await? {
6986                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6987                GateFix::Stop => break,
6988                GateFix::Defer => {
6989                    self.state.save()?;
6990                    return Ok(());
6991                }
6992            }
6993        }
6994        let passed = outcomes.iter().all(CommandOutcome::ok);
6995        self.state.gate = outcomes;
6996        self.state.gate_ran = true;
6997        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6998        if !passed {
6999            self.state.status = RunStatus::Blocked;
7000            let spent = self.state.gate_fixes.len();
7001            self.state.event(
7002                "gate",
7003                if spent == 0 {
7004                    "gate failed; not merging".to_owned()
7005                } else {
7006                    format!("gate failed after {spent} gate-fix round(s); not merging")
7007                },
7008            );
7009        }
7010        self.state.save()?;
7011        Ok(())
7012    }
7013
7014    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7015    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7016    /// after review is clean and never on a candidate awaiting judging.
7017    ///
7018    /// Never fails the run: a non-zero exit or timeout is a warning and a
7019    /// recorded outcome, and the gate remains the single arbiter. Nothing
7020    /// configured means nothing happens - no event, no commit. `commit_all`
7021    /// commits any leftover change under the neutral identity and returns
7022    /// `false` when the tree is clean, so no empty commit is ever made.
7023    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7024        let commands = state.config.verify.pre_gate.clone();
7025        if commands.is_empty() {
7026            return;
7027        }
7028        let shell = state.config.shell();
7029        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7030        let (outcomes, _) = run_commands(
7031            state,
7032            "pre_gate",
7033            "pre_gate",
7034            0,
7035            &shell,
7036            &commands,
7037            &winner.worktree,
7038            timeout,
7039        )
7040        .await;
7041        for o in &outcomes {
7042            if !o.ok() {
7043                tracing::warn!(
7044                    "pre_gate `{}` failed ({:?}); the gate decides",
7045                    o.command,
7046                    o.code
7047                );
7048            }
7049            state.event(
7050                "pre_gate",
7051                format!(
7052                    "`{}` -> {}",
7053                    o.command,
7054                    if o.ok() {
7055                        "pass".to_owned()
7056                    } else {
7057                        format!(
7058                            "FAIL ({:?})\n{}",
7059                            o.code,
7060                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7061                        )
7062                    }
7063                ),
7064            );
7065        }
7066        state.pre_gate = outcomes;
7067        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7068            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7069                Ok(head) => {
7070                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7071                    state.pre_gate_commit = Some(head);
7072                }
7073                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7074            },
7075            Ok(false) => {}
7076            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7077        }
7078        if let Err(e) = state.save() {
7079            tracing::warn!("could not persist the pre_gate record: {e:#}");
7080        }
7081    }
7082
7083    /// Run `verify.gate` once against the winner's current tree, logging one
7084    /// event per command. Empty when nothing is configured.
7085    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7086        let wait = self.repository_wait();
7087        run_landing_gate(&mut self.state, winner, &wait).await
7088    }
7089
7090    /// One bounded fix round for a failing gate.
7091    ///
7092    /// The fixer is told the failure came from the gate itself, not from a
7093    /// reviewer, and is shown the failed commands, their exit codes and a tail
7094    /// of their output - whatever `[verify].gate` holds, nothing here knows
7095    /// what those commands run. Only a normal non-zero exit that printed
7096    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7097    /// command or a full disk says nothing about the code, and a fixer sent
7098    /// after it can only appease the machine. The round is judged by what git
7099    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7100    /// again before the gate does, so a fix cannot trade a green gate for a
7101    /// red e2e unnoticed.
7102    async fn gate_fix_round(
7103        &mut self,
7104        winner: &Candidate,
7105        outcomes: &[CommandOutcome],
7106    ) -> Result<GateFix> {
7107        let cap = self.state.config.graph.gate_fix_rounds;
7108        let spent = self.state.gate_fixes.len();
7109        if spent >= cap {
7110            if cap > 0 {
7111                self.state.event(
7112                    "gate",
7113                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7114                );
7115            }
7116            return Ok(GateFix::Stop);
7117        }
7118        if !gate_fixable(outcomes) {
7119            self.state.event(
7120                "gate",
7121                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7122                 command or similar); not spending a fix round on it",
7123            );
7124            return Ok(GateFix::Stop);
7125        }
7126        let min_free = self.state.config.disk.min_free_bytes;
7127        if min_free > 0 {
7128            match crate::disk::free_bytes(&winner.worktree) {
7129                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7130                Ok(free) => {
7131                    self.state.event(
7132                        "gate",
7133                        format!(
7134                            "only {free} bytes free ({min_free} required by `[disk] \
7135                             min_free_bytes`); not spending a fix round on a failure the disk \
7136                             may explain"
7137                        ),
7138                    );
7139                    return Ok(GateFix::Stop);
7140                }
7141                Err(e) => {
7142                    self.state.event(
7143                        "gate",
7144                        format!("free disk space could not be measured ({e:#}); no fix round"),
7145                    );
7146                    return Ok(GateFix::Stop);
7147                }
7148            }
7149        }
7150
7151        let attempt = spent + 1;
7152        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7153        let base = self.landing_base();
7154        let fix_prompt = prompt::gate_fix_in(
7155            &self.state.instruction,
7156            &failed,
7157            attempt,
7158            cap,
7159            &self.state.config.graph.language,
7160            &self.state.config.graph.github_language,
7161        );
7162        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7163        let sessions = self.state.config.graph.sessions;
7164        let artifacts = self.merge_repair_artifacts();
7165        self.state.event(
7166            "gate",
7167            format!("gate failed; gate-fix round {attempt} of {cap}"),
7168        );
7169        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7170        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7171        // Persist before invoking the fixer: it may commit and then be
7172        // interrupted before its result (or the following e2e) is saved.
7173        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7174            head: before.clone(),
7175            passed: false,
7176            outcomes: Vec::new(),
7177        });
7178        self.state.save()?;
7179        let (_, seat, out) = self
7180            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7181                prompt: fix_prompt.clone(),
7182                spec,
7183                seat,
7184                cwd: winner.worktree.clone(),
7185                timeout,
7186                allow_write: true,
7187                sessions,
7188                artifacts: artifacts.clone(),
7189                stem: format!("gate-fix-{attempt}"),
7190                handover: None,
7191            })
7192            .await;
7193        // The fixer was never asked: no round is spent (nothing is recorded)
7194        // and a later reentry asks the same agent again.
7195        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7196            return Ok(GateFix::Defer);
7197        }
7198        let mut record = GateFixRecord {
7199            agent: seat.agent.clone(),
7200            failed,
7201            notes: String::new(),
7202            committed: false,
7203            error: None,
7204        };
7205        match out {
7206            AgentOutcome::Ok(o) => {
7207                // A missing report is not a failed fix: the round is judged
7208                // by the tree below, and the report only carries prose.
7209                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7210                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7211                }
7212            }
7213            AgentOutcome::Dropped(_) => {
7214                record.error = Some("the CLI dropped the stream".to_owned());
7215            }
7216            AgentOutcome::Quota(o) => {
7217                self.state.quota.push(QuotaLoss {
7218                    seat: seat.key.clone(),
7219                    node: "gate-fix".to_owned(),
7220                    at: Timestamp::now(),
7221                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7222                });
7223                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7224            }
7225            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7226        }
7227        self.state.seats.insert(seat.key.clone(), seat);
7228        let coordination = self.coordinate(&winner.worktree).await?;
7229        if let Ok(r) = git::rescue_commit(
7230            &winner.worktree,
7231            &format!("magi: gate fix {attempt} (uncommitted work)"),
7232        )
7233        .await
7234        {
7235            self.state.note_withheld("gate-fix", &r.withheld);
7236        }
7237        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7238        record.committed = after != before;
7239        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7240        let note = record.error.clone();
7241        self.state.gate_fixes.push(record);
7242        self.state.save()?;
7243        if !changed {
7244            self.state.event(
7245                "gate",
7246                match note {
7247                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7248                    None => format!("gate-fix round {attempt}: the tree did not change"),
7249                },
7250            );
7251            return Ok(GateFix::Stop);
7252        }
7253        self.state.event(
7254            "gate",
7255            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7256        );
7257
7258        drop(coordination);
7259        let wait = self.repository_wait();
7260        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7261    }
7262
7263    // ---------------------------------------------------------------- merge
7264
7265    /// One read-only rewrite by the summary's author; a text the gate still
7266    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7267    /// applies on their word or on silence. `None` means the run is parked on
7268    /// that question and nothing may be pushed or opened yet.
7269    async fn guarded_pr_message(
7270        &mut self,
7271        winner: &Candidate,
7272        facts: Option<&BranchFacts>,
7273        posting: bool,
7274    ) -> Result<Option<PrMessage>> {
7275        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7276        let head = match final_head.as_deref() {
7277            Some(h) => crate::pr_outcome::FinalHead::At(h),
7278            None => crate::pr_outcome::FinalHead::Unknown,
7279        };
7280        // Only a pull request that is about to be posted pays for a summary.
7281        let summarized = if posting {
7282            Some(self.summarize_change(winner, final_head.as_deref()).await)
7283        } else {
7284            None
7285        };
7286        // A busy shared resource is not a permanent cause: publishing an
7287        // "unavailable" description would stick (a later head move never
7288        // edits the body). Stop before anything is pushed or opened; the
7289        // run is `Blocked` and a resume asks the summary again.
7290        if self.state.resource_block.is_some() {
7291            self.state.event(
7292                "pr-summary",
7293                "a shared resource was busy; nothing was published, resume to summarize again",
7294            );
7295            return Ok(None);
7296        }
7297        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7298            anyhow::bail!(
7299                "the final commit moved during public description generation; retry before publishing"
7300            );
7301        }
7302        let note = match (&summarized, &self.state.change_summary) {
7303            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7304            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7305            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7306            (None, _) => ChangeNote::NotAsked,
7307        };
7308        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7309        if !posting {
7310            let identity = crate::scrub::Identity::current();
7311            return Ok(Some(PrMessage {
7312                title: crate::scrub::scrub(&pr.title, &identity),
7313                body: crate::scrub::scrub(&pr.body, &identity),
7314            }));
7315        }
7316        // This very text was already rejected and asked about: no second
7317        // judge call, rewrite or question, just carry the record on.
7318        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7319        if let Some(known) = self
7320            .state
7321            .github_text
7322            .clone()
7323            .filter(|g| g.fingerprint == raw_fp)
7324        {
7325            return self.settle_withheld(winner, &pr, known).await;
7326        }
7327        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7328        let mut violations =
7329            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7330        let identity = crate::scrub::Identity::current();
7331        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7332            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7333            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7334        {
7335            violations.push(crate::github_text::Violation::SensitiveData);
7336        }
7337        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7338        // only a language violation starts one; sensitive data alone goes
7339        // straight to the owner, with the redacted text as the fallback.
7340        let has_language = violations
7341            .iter()
7342            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7343        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7344            self.state.event(
7345                "github-text",
7346                if has_language {
7347                    format!("description rejected: {violations:?}; requesting one rewrite")
7348                } else {
7349                    format!("description rejected: {violations:?}; asking the owner")
7350                },
7351            );
7352            let mut rewritten = false;
7353            let mut rejected_rewrite = None;
7354            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7355            {
7356                let output = text;
7357                if let Some(title) = summary_title(&output) {
7358                    // Judge the redacted candidate that will actually be posted.
7359                    // Sensitive data alone must not discard a useful rewrite.
7360                    let raw_body = summary_without_title(&output);
7361                    let clean_title = crate::scrub::scrub(&title, &identity);
7362                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7363                    let redacted = clean_title != title || body != raw_body;
7364                    let title = clean_title;
7365                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7366                    if !complete {
7367                        self.state.event(
7368                            "github-text",
7369                            "description rewrite omitted required findings",
7370                        );
7371                    }
7372                    if !body.trim().is_empty() {
7373                        body.push_str("\n\n");
7374                        body.push_str(&public_pr_evidence(&self.state, head));
7375                    }
7376                    let marker = format!("magi:run/{}", self.state.id);
7377                    if !body.trim().is_empty() && !body.contains(&marker) {
7378                        body.push_str(&format!("\n\n{marker}"));
7379                    }
7380                    let rewrite = PrMessage {
7381                        title: title.clone(),
7382                        body: body.clone(),
7383                    };
7384                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7385                    let mut failed = crate::github_text::check_for(
7386                        &self.state.config.graph,
7387                        &title,
7388                        &body,
7389                        redecision,
7390                    );
7391                    if (!crate::github_text::shareable(&title)
7392                        || !crate::github_text::shareable(&body))
7393                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7394                    {
7395                        failed.push(crate::github_text::Violation::SensitiveData);
7396                    }
7397                    if complete
7398                        && !title.trim().is_empty()
7399                        && !body.trim().is_empty()
7400                        && !failed.is_empty()
7401                    {
7402                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7403                            title: title.clone(),
7404                            body: body.clone(),
7405                            failed: failed
7406                                .iter()
7407                                .map(|v| crate::github_text::category(*v).to_owned())
7408                                .collect(),
7409                        });
7410                    }
7411                    if complete
7412                        && !title.trim().is_empty()
7413                        && !raw_body.trim().is_empty()
7414                        && failed.is_empty()
7415                    {
7416                        decision = redecision;
7417                        pr = PrMessage { title, body };
7418                        rewritten = true;
7419                        self.state
7420                            .event("github-text", "description rewrite passed");
7421                        if redacted {
7422                            self.state.event(
7423                                "github-text",
7424                                "sensitive data removed from description rewrite",
7425                            );
7426                        }
7427                    }
7428                }
7429            }
7430            if !rewritten {
7431                self.state.event(
7432                    "github-text",
7433                    "description rewrite unavailable or rejected; asking the owner",
7434                );
7435                if let Some(w) =
7436                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7437                {
7438                    // A different text than the one asked about before: that
7439                    // question no longer describes anything.
7440                    self.retire_github_text_question("the withheld text changed");
7441                    let record = crate::run::GithubTextAsk {
7442                        rewrite: rejected_rewrite,
7443                        fingerprint: raw_fp,
7444                        title: w.title,
7445                        body: w.body,
7446                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7447                        question: None,
7448                        asks: 0,
7449                        resolved: false,
7450                        chosen_title: None,
7451                        choice: None,
7452                    };
7453                    return self.settle_withheld(winner, &pr, record).await;
7454                }
7455            }
7456        }
7457        let (title, body) =
7458            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7459        Ok(Some(PrMessage { title, body }))
7460    }
7461
7462    /// Describe the whole change from its final diff in either run mode.
7463    ///
7464    /// The summary is tied to the final commit and to the merge base of the
7465    /// resolved remote base; a record for any other pair is dropped and the
7466    /// agent asked again, so a description never outlives the diff it was
7467    /// written from. The seat is read-only and the prompt tells it to read the
7468    /// diff and the code around it. `[roles] synthesizer` selects the agents
7469    /// (unset: the default pick), each asked at most once, advancing on an
7470    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7471    /// carries the reason it is unavailable; nothing is invented then.
7472    async fn summarize_change(
7473        &mut self,
7474        winner: &Candidate,
7475        head: Option<&str>,
7476    ) -> std::result::Result<(), String> {
7477        let Some(head) = head else {
7478            return Err("the final commit could not be read".to_owned());
7479        };
7480        let remote = self.state.config.merge.remote.clone();
7481        let tracking = format!("{remote}/{}", self.state.base_branch);
7482        if !git::rev_exists(&winner.worktree, &tracking).await {
7483            return Err(format!("the remote base `{tracking}` could not be read"));
7484        }
7485        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7486            Ok(b) if !b.is_empty() => b,
7487            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7488        };
7489        let evidence = crate::pr_outcome::summary_evidence(
7490            &self.state,
7491            crate::pr_outcome::FinalHead::At(head),
7492        );
7493        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7494        if self
7495            .state
7496            .change_summary
7497            .as_ref()
7498            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7499        {
7500            return Ok(());
7501        }
7502        if self.state.change_summary.take().is_some() {
7503            self.state.event(
7504                "pr-summary",
7505                "the final diff or evidence changed; the earlier change summary was discarded",
7506            );
7507        }
7508        // Save the source snapshot even if no generation agent is available.
7509        if let Err(e) = write_artifact(
7510            &self.state,
7511            &format!("public-description-{fingerprint}.md"),
7512            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7513        ) {
7514            tracing::warn!("could not save public description evidence: {e:#}");
7515        }
7516        let chain = match agent::pick_chain(
7517            &self.state.config.agents,
7518            self.state.config.roles.synthesizer.as_ref(),
7519            &agent::installed,
7520            "synthesizer",
7521        ) {
7522            Ok(chain) => chain,
7523            Err(e) => {
7524                tracing::warn!("no agent for the change summary: {e:#}");
7525                self.state.event("pr-summary", "no summary agent available");
7526                return Err("no agent was available to summarize it".to_owned());
7527            }
7528        };
7529        let subjects = git::subjects(&winner.worktree, &base, head)
7530            .await
7531            .unwrap_or_default();
7532        let stat = git::diff_stat(&winner.worktree, &base, head)
7533            .await
7534            .unwrap_or_default();
7535        let stat: String = stat.chars().take(6_000).collect();
7536        let mut prompt = crate::prompt::change_summary_in(
7537            &base,
7538            head,
7539            &subjects,
7540            stat.trim(),
7541            &self.state.config.graph.github_language,
7542        );
7543        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7544        let sessions = self.state.config.graph.sessions;
7545        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7546        let prompts = self.state.config.prompts.clone();
7547        let cache = self.state.config.cache_dir();
7548        let run = self.state.id.clone();
7549        for (i, spec) in chain.into_iter().enumerate() {
7550            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7551            let job = SeatJob {
7552                spec: spec.clone(),
7553                seat,
7554                cwd: winner.worktree.clone(),
7555                prompt: prompt.clone(),
7556                timeout,
7557                allow_write: false,
7558                sessions,
7559                artifacts: agent::artifacts_dir(&self.state.dir()),
7560                stem: if i == 0 {
7561                    "change-summary".to_owned()
7562                } else {
7563                    format!("change-summary-{}", spec.id)
7564                },
7565                handover: None,
7566            };
7567            let ctx = WaveCtx {
7568                run: &run,
7569                node: "pr-summary",
7570                prompts: &prompts,
7571                cache: cache.as_deref(),
7572                round: None,
7573                carry_seats: false,
7574                pauses: Some((&self.pause, &self.interrupt)),
7575            };
7576            let (seat, outcome) =
7577                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7578            self.state.seats.insert(seat.key.clone(), seat);
7579            let text = match outcome {
7580                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7581                AgentOutcome::ResourceBlocked(why) => {
7582                    // No agent was asked: the run stops resumably before
7583                    // anything is published (see `guarded_pr_message`).
7584                    if self.state.resource_block.is_none() {
7585                        self.state.resource_block = Some(crate::run::ResourceBlock {
7586                            node: "pr-summary".to_owned(),
7587                            kind: "shared resource".to_owned(),
7588                            detail: why,
7589                            at: Timestamp::now(),
7590                        });
7591                    }
7592                    return Err("a shared resource was busy".to_owned());
7593                }
7594                _ => continue,
7595            };
7596            let blind = &self.state.config.blind;
7597            let Some(title) = summary_title(&text) else {
7598                continue;
7599            };
7600            let body: String = summary_without_title(&text)
7601                .lines()
7602                .filter(|l| !l.contains("magi:run/"))
7603                .collect::<Vec<_>>()
7604                .join("\n");
7605            let body = blind::sanitize_prose(body.trim(), blind);
7606            let title = blind::sanitize_prose(&title, blind);
7607            if body.trim().is_empty() || title.trim().is_empty() {
7608                continue;
7609            }
7610            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7611                continue;
7612            }
7613            // The branch must still be the one the agent was shown.
7614            if git::rev_parse(&winner.worktree, "HEAD")
7615                .await
7616                .ok()
7617                .as_deref()
7618                != Some(head)
7619            {
7620                return Err("the branch moved while the summary was written".to_owned());
7621            }
7622            if git::merge_base(&winner.worktree, &tracking, head)
7623                .await
7624                .ok()
7625                .as_deref()
7626                != Some(base.as_str())
7627                || crate::pr_outcome::summary_evidence(
7628                    &self.state,
7629                    crate::pr_outcome::FinalHead::At(head),
7630                ) != evidence
7631            {
7632                return Err("the summary evidence changed while it was written".to_owned());
7633            }
7634            self.state.change_summary = Some(crate::run::ChangeSummary {
7635                head: head.to_owned(),
7636                base,
7637                evidence: fingerprint,
7638                title,
7639                body,
7640            });
7641            self.state.event(
7642                "pr-summary",
7643                format!("change summary written by {} for the final diff", spec.id),
7644            );
7645            if let Err(e) = self.state.save() {
7646                tracing::warn!("could not save the change summary: {e:#}");
7647            }
7648            return Ok(());
7649        }
7650        self.state
7651            .event("pr-summary", "no agent produced a change summary");
7652        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7653    }
7654
7655    /// One read-only rewrite request for a withheld description. The
7656    /// implementer's own conversation is reused only when it exists and can be
7657    /// resumed; otherwise the `[roles] language_judge` chain (or the default
7658    /// pick) writes it in a separate conversation. The prompt carries the whole
7659    /// text, so no agent needs any memory. Each agent is asked at most once and
7660    /// the chain advances on an error, quota, unusable answer or a reply with
7661    /// no `TITLE:` line. Nothing is recorded as a handover.
7662    async fn rewrite_pr_text(
7663        &mut self,
7664        winner: &Candidate,
7665        pr: &PrMessage,
7666        violations: &[crate::github_text::Violation],
7667    ) -> Option<String> {
7668        let prompt = format!(
7669            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7670            pr.title, pr.body
7671        );
7672        let sessions = self.state.config.graph.sessions;
7673        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7674        // (spec, seat, resumed)
7675        let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7676        let key = format!("impl-{}", winner.label);
7677        if let Some(spec) = self
7678            .state
7679            .config
7680            .agents
7681            .iter()
7682            .find(|a| a.id == winner.agent)
7683            .cloned()
7684            && let Some(existing) = self.state.seats.get(&key)
7685            && existing.agent == spec.id
7686            && agent::has_session(spec.kind, existing, sessions)
7687        {
7688            attempts.push((spec, existing.clone(), true));
7689        } else {
7690            let chain = agent::pick_chain(
7691                &self.state.config.agents,
7692                self.state.config.roles.language_judge.as_ref(),
7693                &agent::installed,
7694                "language_judge",
7695            );
7696            match chain {
7697                Ok(chain) => {
7698                    for spec in chain {
7699                        let seat = crate::fixer::seat_for(
7700                            &mut self.state,
7701                            "github-text-rewrite",
7702                            &spec.id,
7703                        );
7704                        attempts.push((spec, seat, false));
7705                    }
7706                }
7707                Err(e) => {
7708                    tracing::warn!("no agent for the description rewrite: {e:#}");
7709                    self.state
7710                        .event("github-text", "no rewrite agent available");
7711                    return None;
7712                }
7713            }
7714        }
7715        let prompts = self.state.config.prompts.clone();
7716        let cache = self.state.config.cache_dir();
7717        let run = self.state.id.clone();
7718        for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7719            let job = SeatJob {
7720                spec: spec.clone(),
7721                seat,
7722                cwd: winner.worktree.clone(),
7723                prompt: prompt.clone(),
7724                timeout: retry_budget(timeout_full, resumed),
7725                allow_write: false,
7726                sessions,
7727                artifacts: agent::artifacts_dir(&self.state.dir()),
7728                stem: if i == 0 {
7729                    "github-text-rewrite".to_owned()
7730                } else {
7731                    format!("github-text-rewrite-{}", spec.id)
7732                },
7733                handover: None,
7734            };
7735            let ctx = WaveCtx {
7736                run: &run,
7737                node: "github-text",
7738                prompts: &prompts,
7739                cache: cache.as_deref(),
7740                round: None,
7741                carry_seats: false,
7742                pauses: Some((&self.pause, &self.interrupt)),
7743            };
7744            let (seat, outcome) =
7745                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7746            self.state.seats.insert(seat.key.clone(), seat);
7747            match outcome {
7748                AgentOutcome::Ok(o)
7749                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7750                {
7751                    return Some(o.text);
7752                }
7753                AgentOutcome::ResourceBlocked(_) => return None,
7754                _ => {}
7755            }
7756        }
7757        None
7758    }
7759
7760    /// Abandon the standing posting-gate question, if any.
7761    fn retire_github_text_question(&mut self, why: &str) {
7762        let Some(id) = self
7763            .state
7764            .github_text
7765            .as_ref()
7766            .and_then(|g| g.question.clone())
7767        else {
7768            return;
7769        };
7770        let _ = ask::Questions::open().update(&id, |q| {
7771            q.abandon(why);
7772            Ok(())
7773        });
7774    }
7775
7776    /// Carry a withheld text to its end: file the question, park while it is
7777    /// open, and on an answer, silence or timeout pick the text to post. The
7778    /// decision is saved before it is used, so it applies once.
7779    async fn settle_withheld(
7780        &mut self,
7781        winner: &Candidate,
7782        pr: &PrMessage,
7783        mut g: crate::run::GithubTextAsk,
7784    ) -> Result<Option<PrMessage>> {
7785        use crate::github_text as gt;
7786        let store = ask::Questions::open();
7787        let timeout = self.state.config.graph.answer_timeout;
7788        if !g.resolved {
7789            let mut standing = g
7790                .question
7791                .as_ref()
7792                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7793            if let Some(q) = standing.as_ref()
7794                && q.status.open()
7795                && gt::expired(q, timeout)
7796            {
7797                let why = format!("no answer within {}s of asking", timeout.max(1));
7798                standing = store
7799                    .update(&q.id, |q| {
7800                        q.abandon(&why);
7801                        Ok(())
7802                    })
7803                    .ok()
7804                    .map(|(q, ())| q);
7805            }
7806            let mut ask_again: Option<Option<Vec<String>>> = None;
7807            match standing {
7808                // Recorded but never written (a stop between the two saves),
7809                // or removed: ask again without spending the retry.
7810                None if g.asks <= 1 => {
7811                    g.asks = 0;
7812                    ask_again = Some(None);
7813                }
7814                None => {
7815                    g.resolved = true;
7816                    g.chosen_title = None;
7817                    g.choice = Some(GithubTextChoice::Fallback);
7818                }
7819                Some(q) if q.status.open() => {
7820                    self.park_on_github_text(&g)?;
7821                    return Ok(None);
7822                }
7823                Some(q) => match gt::read_reply(&q) {
7824                    gt::Reply::Fallback => {
7825                        g.resolved = true;
7826                        g.chosen_title = None;
7827                        g.choice = Some(GithubTextChoice::Fallback);
7828                    }
7829                    gt::Reply::Original => {
7830                        g.resolved = true;
7831                        g.chosen_title = None;
7832                        g.choice = Some(GithubTextChoice::Original);
7833                    }
7834                    gt::Reply::Rewrite => {
7835                        g.resolved = true;
7836                        g.chosen_title = None;
7837                        g.choice = Some(if g.rewrite.is_some() {
7838                            GithubTextChoice::Rewrite
7839                        } else {
7840                            GithubTextChoice::Fallback
7841                        });
7842                    }
7843                    gt::Reply::Title(text) => {
7844                        let first = text
7845                            .lines()
7846                            .map(str::trim)
7847                            .find(|l| !l.is_empty())
7848                            .unwrap_or("");
7849                        let decision = if first.is_empty() {
7850                            None
7851                        } else {
7852                            let candidate = PrMessage {
7853                                title: first.to_owned(),
7854                                body: String::new(),
7855                            };
7856                            self.judge_pr_language(&winner.worktree, &candidate).await
7857                        };
7858                        match gt::vet_title(
7859                            &text,
7860                            decision,
7861                            self.state.config.graph.github_english(),
7862                        ) {
7863                            Ok(title) => {
7864                                g.resolved = true;
7865                                g.chosen_title = Some(title);
7866                                g.choice = Some(GithubTextChoice::MyTitle);
7867                            }
7868                            Err(cats) if g.asks < 2 => {
7869                                ask_again = Some(Some(
7870                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7871                                ))
7872                            }
7873                            Err(_) => {
7874                                g.resolved = true;
7875                                g.chosen_title = None;
7876                                g.choice = Some(GithubTextChoice::Fallback);
7877                            }
7878                        }
7879                    }
7880                },
7881            }
7882            if let Some(rejected) = ask_again {
7883                // A retry (after a rejected replacement title) is still about
7884                // the original text: the same ORIGINAL / FALLBACK comparison.
7885                let w = gt::Withheld {
7886                    title: g.title,
7887                    body: g.body,
7888                    categories: g.categories.clone(),
7889                };
7890                // Named by the original text's fingerprint alone, so a resume
7891                // rewrites the same file and the detail stays identical.
7892                let artifact = match crate::run::write_artifact(
7893                    &self.state,
7894                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7895                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7896                ) {
7897                    Ok(p) => Some(p),
7898                    Err(e) => {
7899                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7900                        None
7901                    }
7902                };
7903                let (detail, comparison) = gt::question_content(
7904                    &w,
7905                    &pr.title,
7906                    &pr.body,
7907                    &self.state.id,
7908                    rejected.as_deref(),
7909                    artifact.as_deref(),
7910                    g.rewrite.as_ref(),
7911                );
7912                let mut q = ask::Question::new(
7913                    self.state.id.clone(),
7914                    gt::ASK_NODE.to_owned(),
7915                    gt::ASK_SEAT.to_owned(),
7916                    gt::question_summary(&self.state.config.graph.language, &w),
7917                    detail,
7918                    gt::question_choices(&w, g.rewrite.as_ref()),
7919                );
7920                q.posting_comparison = Some(comparison);
7921                q.answer_timeout = timeout;
7922                // Recorded before it is written: a stop in between leaves a
7923                // record naming a question the resume then files again.
7924                g.question = Some(q.id.clone());
7925                g.asks += 1;
7926                self.state.github_text = Some(g.clone());
7927                self.state.event(
7928                    "github-text",
7929                    format!("asking the owner about the withheld text ({})", q.short()),
7930                );
7931                self.state.save()?;
7932                store
7933                    .put_executable(&mut q)
7934                    .context("file the posting-gate question")?;
7935                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7936                    tracing::warn!(
7937                        "could not notify about posting-gate question {}: {e:#}",
7938                        q.short()
7939                    );
7940                }
7941                self.park_on_github_text(&g)?;
7942                return Ok(None);
7943            }
7944        }
7945        let resolved = g.chosen_title.clone();
7946        let original = g.choice == Some(GithubTextChoice::Original);
7947        // Field names only: the event log never records the text.
7948        self.state.event(
7949            "github-text",
7950            if original {
7951                format!(
7952                    "posting the original text for: {}",
7953                    match (g.title, g.body) {
7954                        (true, true) => "title, description",
7955                        (true, false) => "title",
7956                        _ => "description",
7957                    }
7958                )
7959            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7960                "posting the saved rewrite title and description".to_owned()
7961            } else if resolved.is_some() {
7962                "posting the owner's replacement title".to_owned()
7963            } else {
7964                "posting the neutral text for the withheld fields".to_owned()
7965            },
7966        );
7967        self.state.github_text = Some(g.clone());
7968        self.state.save()?;
7969        if g.choice == Some(GithubTextChoice::Rewrite)
7970            && let Some(r) = g.rewrite.as_ref()
7971        {
7972            return Ok(Some(PrMessage {
7973                title: r.title.clone(),
7974                body: r.body.clone(),
7975            }));
7976        }
7977        let w = gt::Withheld {
7978            title: g.title,
7979            body: g.body,
7980            categories: g.categories.clone(),
7981        };
7982        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7983        if let Some(t) = resolved {
7984            title = t;
7985        }
7986        // Every withheld field is now neutral or vetted English.
7987        let english = gt::LanguageDecision {
7988            title_english: true,
7989            body_english: true,
7990            body_complete: true,
7991        };
7992        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7993        if original {
7994            // The owner's explicit choice overrides the gate for the fields it
7995            // was made for; the others keep what the gate produced.
7996            if g.title {
7997                title = pr.title.clone();
7998            }
7999            if g.body {
8000                body = pr.body.clone();
8001            }
8002        }
8003        Ok(Some(PrMessage { title, body }))
8004    }
8005
8006    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8007        self.state.github_text = Some(g.clone());
8008        self.state.end_pass_parked(ParkCause::Question);
8009        self.state.parked = true;
8010        self.state.event(
8011            "github-text",
8012            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8013        );
8014        self.state.save()
8015    }
8016
8017    /// Ask `[roles] language_judge` about this text and record which source
8018    /// decided; the text itself is never recorded.
8019    async fn judge_pr_language(
8020        &mut self,
8021        cwd: &Path,
8022        pr: &PrMessage,
8023    ) -> Option<crate::github_text::LanguageDecision> {
8024        self.state.config.roles.language_judge.as_ref()?;
8025        if !self.state.config.graph.language_gate() {
8026            return None;
8027        }
8028        let decision =
8029            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8030        self.state.event(
8031            "github-text",
8032            if decision.is_some() {
8033                "language decided by the language judge"
8034            } else {
8035                "language judge unavailable; using built-in heuristics"
8036            },
8037        );
8038        decision
8039    }
8040
8041    async fn merge(&mut self) -> Result<()> {
8042        // Same reasoning as `gate`: ask the review and gate records directly
8043        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8044        // skip can rewrite on reentry to something that no longer says
8045        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8046        // so a hand-off (open findings, green verification) reaches merge
8047        // exactly like a genuinely clean round does.
8048        //
8049        // A run resumed mid-`land` never reaches here at all: `execute`
8050        // recognises `RunStatus::Landing` before it even calls `prep`, and
8051        // routes straight to `run_land` instead. That has to happen a level
8052        // up from this function, not with a check in here, because
8053        // `review_loop`'s own status recomputation (see its doc) runs
8054        // *before* `merge` on every reentry and would otherwise overwrite
8055        // the `Landing` marker with `Gating` before this node ever saw it.
8056        if self
8057            .state
8058            .base_sync
8059            .as_ref()
8060            .is_some_and(|s| s.conflict.is_some())
8061            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8062                != Some(RunStatus::Gating)
8063            // `gate_ran == false` is not "passed" - `gate` leaves it false
8064            // both before it has ever run and when its last attempt was
8065            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8066            // permission to merge on nothing but the review record. Only a
8067            // gate that actually ran may proceed; an intentional local
8068            // review can also finish with verification explicitly not configured.
8069            // `RunState::gate_status` computes that distinction.
8070            || !self.state.gate_allows_progress()
8071        {
8072            return Ok(());
8073        }
8074        // This node's own record, not `status`: `status == Ready` is not
8075        // unique to the harmless `MergeMode::None` path this line was
8076        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8077        // run's PR was closed without merging — and on that run `mode` is
8078        // still `Pr`, so a reentry that fell through here would push and
8079        // open a second pull request. `self.state.merge` is set exactly once
8080        // this node (or `land`) has already produced a verdict, under every
8081        // mode, which is what "already done" actually means here.
8082        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8083            return Ok(());
8084        }
8085        let Some(winner) = self.state.winner().cloned() else {
8086            return Ok(());
8087        };
8088        let repo = self.state.repo.clone();
8089        let base = self.state.base_branch.clone();
8090        let mode = self.state.config.merge.mode;
8091        let style = self.state.config.merge.style;
8092        let facts = if is_review_run(&self.state) {
8093            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8094            let start = review_base(
8095                &repo,
8096                &self.state.config.merge.remote,
8097                &base,
8098                &self.state.base_commit,
8099                &winner.branch,
8100            )
8101            .await;
8102            branch_facts(&repo, &start, &winner.branch).await
8103        } else {
8104            None
8105        };
8106        // `None` when the base could not be freshly read: then an adopted
8107        // pull request's title is left alone.
8108        let leaked = if is_review_run(&self.state) {
8109            leaked_subjects(&self.state, &winner.branch).await
8110        } else {
8111            Some(Vec::new())
8112        };
8113        // `None`: parked on the owner's word about a withheld title or body.
8114        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8115        // the resume arrives here again.
8116        let Some(pr) = self
8117            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8118            .await?
8119        else {
8120            return Ok(());
8121        };
8122        let message = pr.commit_message();
8123
8124        let outcome = match mode {
8125            MergeMode::None => MergeOutcome {
8126                mode,
8127                ok: true,
8128                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8129                empty: false,
8130            },
8131            MergeMode::Pr | MergeMode::Local
8132                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8133            {
8134                MergeOutcome {
8135                    mode,
8136                    ok: false,
8137                    detail: empty_candidate_detail(&self.state, &base),
8138                    empty: true,
8139                }
8140            }
8141            MergeMode::Local => {
8142                let on = git::current_branch(&repo).await?;
8143                if on.as_deref() != Some(base.as_str()) {
8144                    MergeOutcome {
8145                        mode,
8146                        ok: false,
8147                        detail: format!(
8148                            "{} has {} checked out, not the base branch {base}",
8149                            repo.display(),
8150                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8151                        ),
8152                        empty: false,
8153                    }
8154                } else if !git::is_clean(&repo).await? {
8155                    MergeOutcome {
8156                        mode,
8157                        ok: false,
8158                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8159                        empty: false,
8160                    }
8161                } else {
8162                    let out = match style {
8163                        MergeStyle::Merge => {
8164                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8165                        }
8166                        MergeStyle::Squash => {
8167                            git::merge_squash(&repo, &winner.branch, &message).await?
8168                        }
8169                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8170                    };
8171                    MergeOutcome {
8172                        mode,
8173                        ok: out.ok(),
8174                        detail: if out.ok() { out.stdout } else { out.stderr },
8175                        empty: false,
8176                    }
8177                }
8178            }
8179            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8180                self.state.pr_publication_pending = true;
8181                self.state.save()?;
8182                let out =
8183                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8184                self.state.pr_publication_pending = out.is_err();
8185                MergeOutcome {
8186                    mode,
8187                    ok: out.is_ok(),
8188                    detail: match out {
8189                        Ok(url) => url,
8190                        Err(e) => format!("{e:#}"),
8191                    },
8192                    empty: false,
8193                }
8194            }
8195            MergeMode::Pr => {
8196                let remote = self.state.config.merge.remote.clone();
8197                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8198                if !pushed.ok() {
8199                    MergeOutcome {
8200                        mode,
8201                        ok: false,
8202                        detail: pushed.stderr,
8203                        empty: false,
8204                    }
8205                } else {
8206                    // A retry or resume of a run whose branch already has an
8207                    // open pull request adopts it rather than failing on a
8208                    // duplicate. Only this winner branch into this base:
8209                    // `branch_for` derives the name from the run id, so a
8210                    // different run's pull request never matches.
8211                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8212                    let out = match pr_merge_plan(found) {
8213                        PrPlan::Create => {
8214                            gh_pr_create(
8215                                &winner.worktree,
8216                                &base,
8217                                &winner.branch,
8218                                &pr.title,
8219                                &pr.body,
8220                            )
8221                            .await
8222                        }
8223                        PrPlan::Adopt { url, title } => {
8224                            self.state
8225                                .event("merge", format!("Pr: adopted open pull request {url}"));
8226                            if self.state.github_text.as_ref().is_some_and(|g| {
8227                                g.resolved
8228                                    && g.choice == Some(GithubTextChoice::Rewrite)
8229                                    && g.rewrite.is_some()
8230                            }) {
8231                                // The owner chose the entire pair, even if the
8232                                // existing title already matches or was edited.
8233                                land::set_pr_text_verbatim(
8234                                    &winner.worktree,
8235                                    &url,
8236                                    &pr.title,
8237                                    &pr.body,
8238                                )
8239                                .await
8240                                .map(|()| url)
8241                            } else {
8242                                if title != pr.title
8243                                    && (!is_review_run(&self.state)
8244                                        || leaked
8245                                            .as_deref()
8246                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8247                                    && let Err(e) =
8248                                        if self.state.github_text.as_ref().is_some_and(|g| {
8249                                            g.resolved
8250                                                && g.title
8251                                                && g.choice == Some(GithubTextChoice::Original)
8252                                        }) {
8253                                            // The owner chose to post this title unmodified.
8254                                            land::set_pr_title_verbatim(
8255                                                &winner.worktree,
8256                                                &url,
8257                                                &pr.title,
8258                                            )
8259                                            .await
8260                                        } else {
8261                                            land::set_pr_title(
8262                                                &mut self.state,
8263                                                &winner.worktree,
8264                                                &url,
8265                                                &pr.title,
8266                                            )
8267                                            .await
8268                                        }
8269                                {
8270                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8271                                    self.state
8272                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8273                                }
8274                                Ok(url)
8275                            }
8276                        }
8277                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8278                    };
8279                    match out {
8280                        Ok(url) => MergeOutcome {
8281                            mode,
8282                            ok: true,
8283                            detail: url,
8284                            empty: false,
8285                        },
8286                        Err(e) => MergeOutcome {
8287                            mode,
8288                            ok: false,
8289                            detail: e.to_string(),
8290                            empty: false,
8291                        },
8292                    }
8293                }
8294            }
8295        };
8296
8297        self.state.status = match (mode, outcome.ok) {
8298            (MergeMode::None, _) => RunStatus::Ready,
8299            (_, true) if outcome.empty => RunStatus::Ready,
8300            (MergeMode::Pr, true) => RunStatus::Ready,
8301            (_, true) => RunStatus::Merged,
8302            (_, false) => RunStatus::Blocked,
8303        };
8304        self.state.event(
8305            "merge",
8306            format!(
8307                "{:?}: {}",
8308                mode,
8309                outcome.detail.lines().next().unwrap_or("")
8310            ),
8311        );
8312        self.state.merge = Some(outcome);
8313        self.state.save()?;
8314
8315        // The PR is open and the run would historically stop here, leaving the
8316        // operator to watch checks, feed review comments back to a fixer, and
8317        // merge. That was done by hand six times in one session before this
8318        // existed. Opt-in, because merging is the one irreversible thing magi
8319        // can do to a repository.
8320        if self.state.config.graph.land
8321            && mode == MergeMode::Pr
8322            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8323        {
8324            self.run_land().await?;
8325        }
8326        // `run_land` may have left `status` at `Landing` - still waiting on
8327        // CI or the owner's approval, not actually settled - so this has to
8328        // read whatever `status` ended up as here, not the `Merged` this
8329        // function set a few lines up.
8330        self.settle_questions();
8331        Ok(())
8332    }
8333
8334    /// Enter `land`.
8335    ///
8336    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8337    /// a resumed run's re-entry. `land::land` itself is what serialises the
8338    /// two git-mutating moments inside the loop — the rebase push and
8339    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8340    /// to hold a lock across the whole call, and doing so would serialise
8341    /// this run's CI wait against a *different* run's land-approval resume
8342    /// in the same repository, which is exactly the "must not wait on
8343    /// another task" property the daemon's slot-freeing exists to give.
8344    async fn run_land(&mut self) -> Result<()> {
8345        crate::pr_head::ensure_owner(&self.state)?;
8346        let url = self
8347            .state
8348            .pr
8349            .as_ref()
8350            .map(|pr| pr.url.clone())
8351            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8352            .unwrap_or_default();
8353        let url = url.lines().next().unwrap_or("").trim().to_owned();
8354        if !url.starts_with("http") {
8355            return Ok(());
8356        }
8357        // A land failure is not a lost run: the work is on a branch and the
8358        // pull request is open, which is exactly where a human takes over.
8359        loop {
8360            match land::land(&mut self.state, &url).await {
8361                Ok(_)
8362                    if self
8363                        .state
8364                        .merge_repairs
8365                        .last()
8366                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8367                {
8368                    if self.repair_merge_review().await? {
8369                        continue;
8370                    }
8371                }
8372                Ok(pr) if self.state.parked => {
8373                    // `land` already saved the parked marker; nothing here
8374                    // overrides `status` back to a terminal value while an
8375                    // approval is still outstanding.
8376                    let _ = pr;
8377                }
8378                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8379                    self.state.status = RunStatus::Superseded;
8380                    self.state.save()?;
8381                }
8382                Ok(pr) => {
8383                    self.state.status = match pr.state {
8384                        land::PrLifecycle::Merged => RunStatus::Merged,
8385                        _ => RunStatus::Blocked,
8386                    };
8387                    // Downstream of a confirmed merge only - see
8388                    // `bump::should_release_bump`'s own doc for why this one
8389                    // check covers all three of `land`'s success paths.
8390                    // Best-effort: the run already landed, so a failure here
8391                    // (the decision call, `gh`, `cargo`) is recorded and never
8392                    // turns a landed run into a failed one.
8393                    if bump::should_release_bump(self.state.status)
8394                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8395                    {
8396                        // The event is the run's own record. Not-eligible cases
8397                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8398                        // `Err` is a bump that was tried and failed:
8399                        // `after_merge` itself raises the operator notice for
8400                        // that, whether or not a release PR exists yet.
8401                        self.state
8402                            .event("bump", format!("release bump skipped: {e:#}"));
8403                    }
8404                    // Independent of the bump, and best-effort in the same way:
8405                    // findings the merge left open become follow-up tasks.
8406                    if self.state.status == RunStatus::Merged {
8407                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8408                    }
8409                    self.state.save()?;
8410                }
8411                Err(e) => {
8412                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8413                        return Err(e);
8414                    }
8415                    self.state.status = RunStatus::Blocked;
8416                    self.state.event("land", format!("gave up: {e}"));
8417                    self.state.save()?;
8418                }
8419            }
8420            break;
8421        }
8422        Ok(())
8423    }
8424
8425    /// One repair cycle stays under the existing driver/task claim. An
8426    /// interrupted fixer is rolled back and held rather than run twice.
8427    async fn repair_merge_review(&mut self) -> Result<bool> {
8428        use crate::run::MergeRepairPhase as Phase;
8429        crate::pr_head::ensure_owner(&self.state)?;
8430        let i = self.state.merge_repairs.len() - 1;
8431        let request = self.state.merge_repairs[i].clone();
8432        if request.phase == Phase::Fixing {
8433            let restored = match self.state.winner().cloned() {
8434                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8435                None => Err(anyhow::anyhow!("repair has no winning branch")),
8436            };
8437            let why = match restored {
8438                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8439                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8440                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8441            };
8442            self.hold_merge_repair(i, why)?;
8443            return Ok(false);
8444        }
8445        let failure = if i >= self.state.config.graph.land_rounds
8446            || self.state.config.graph.review_rounds == 0
8447        {
8448            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8449        } else {
8450            None
8451        };
8452        if let Some(why) = failure {
8453            self.hold_merge_repair(i, why)?;
8454            return Ok(false);
8455        }
8456        if request.phase == Phase::Requested {
8457            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8458                Ok(pr) => pr,
8459                Err(e) => {
8460                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8461                    return Ok(false);
8462                }
8463            };
8464            self.state.merge_repairs[i].phase = Phase::Fixing;
8465            self.state.land_approval = None;
8466            self.state.land_armed_head = None;
8467            self.state.save()?;
8468            // Serialize the exact records, including finding ids/details,
8469            // reconsideration reasons, final votes and verification output.
8470            let evidence = serde_json::to_string_pretty(&request)?;
8471            let reason = format!(
8472                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8473            );
8474            let budget = self.state.config.graph.land_rounds;
8475            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8476            match result {
8477                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8478                Ok(land::Fixed::Failed(why)) => {
8479                    self.hold_merge_repair(
8480                        i,
8481                        format!(
8482                            "repair failed: {why}; inspect the branch and run an explicit review"
8483                        ),
8484                    )?;
8485                    return Ok(false);
8486                }
8487                Err(e) => {
8488                    self.hold_merge_repair(
8489                        i,
8490                        format!(
8491                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8492                        ),
8493                    )?;
8494                    return Ok(false);
8495                }
8496            }
8497            // Even no change gets a real new reviewer wave. The previous
8498            // evidence remains in the request; a former clean/concluded round
8499            // cannot short-circuit the new review or gate.
8500            self.begin_merge_repair_review(i)?;
8501        }
8502        self.review_loop().await?;
8503        self.gate().await?;
8504        if self.resource_halt()? || self.state.parked {
8505            return Ok(false);
8506        }
8507        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8508            != Some(RunStatus::Gating)
8509            || !self.state.gate_allows_progress()
8510        {
8511            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8512            return Ok(false);
8513        }
8514        // Review and gate may have added commits after the initial repair.
8515        // Publish exactly their final head to the existing PR, with a lease.
8516        let winner = self
8517            .state
8518            .winner()
8519            .cloned()
8520            .context("repair has no winning branch")?;
8521        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8522        if self.state.gate_head.as_deref() != Some(after.as_str())
8523            || !git::is_clean(&winner.worktree).await?
8524        {
8525            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8526            return Ok(false);
8527        }
8528        let expected = self
8529            .state
8530            .pr_pushed_head
8531            .as_deref()
8532            .unwrap_or(&request.head)
8533            .to_owned();
8534        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8535        // Push the checked commit itself: a local ref moving after the check
8536        // must not change which commit the forge receives.
8537        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8538        let refspec = format!("{after}:refs/heads/{public}");
8539        let push = git::git_raw(
8540            &winner.worktree,
8541            &["push", &lease, &self.state.config.merge.remote, &refspec],
8542        )
8543        .await;
8544        match push {
8545            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8546            Ok(out) => {
8547                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8548                return Ok(false);
8549            }
8550            Err(e) => {
8551                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8552                return Ok(false);
8553            }
8554        }
8555        self.state.merge_repairs[i].phase = Phase::Complete;
8556        self.state.status = RunStatus::Landing;
8557        self.state.event(
8558            "repair",
8559            "new review and gate completed; fresh owner merge decision required",
8560        );
8561        self.state.save()?;
8562        Ok(true)
8563    }
8564
8565    fn merge_repair_artifacts(&self) -> PathBuf {
8566        let artifacts = agent::artifacts_dir(&self.state.dir());
8567        match self
8568            .state
8569            .merge_repairs
8570            .last()
8571            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8572        {
8573            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8574            None => artifacts,
8575        }
8576    }
8577
8578    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8579        self.state.reviews.clear();
8580        self.state.gate.clear();
8581        self.state.gate_ran = false;
8582        self.state.gate_head = None;
8583        self.state.gate_fixes.clear();
8584        self.state.gate_repair_e2e = None;
8585        self.state.pre_gate.clear();
8586        self.state.pre_gate_commit = None;
8587        self.state.contested_handoff = None;
8588        self.state.land_reruns = None;
8589        self.state.land_approval = None;
8590        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8591        self.state.status = RunStatus::Reviewing;
8592        self.state.save()
8593    }
8594
8595    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8596        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8597        self.state.merge_repairs[index].failure = Some(why.clone());
8598        self.state.status = RunStatus::Blocked;
8599        self.state.event("repair", why);
8600        self.state.save()
8601    }
8602
8603    // -------------------------------------------------------------- helpers
8604
8605    /// Fetch or create a seat, keeping its conversation across nodes.
8606    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8607        if let Some(existing) = self.state.seats.get(key)
8608            && existing.agent == agent
8609        {
8610            return existing.clone();
8611        }
8612        // A seat that changes agent mints its session id from the agent too,
8613        // like a handover: the old agent's uuid is already taken by the CLI.
8614        let fresh = if self.state.seats.contains_key(key) {
8615            handover_seat(key, agent, self.state.next_seat_seed())
8616        } else {
8617            SeatState::new(key, agent, self.state.seed)
8618        };
8619        self.state.seats.insert(key.to_owned(), fresh.clone());
8620        fresh
8621    }
8622
8623    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8624    /// seat to another roster agent, in which case that agent. Nodes that
8625    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8626    /// reconsideration) must keep talking to whoever answered it, not slip
8627    /// back to the agent that failed it.
8628    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8629        match self.state.seats.get(key) {
8630            Some(s) if s.agent != spec.id => {
8631                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8632            }
8633            _ => spec,
8634        }
8635    }
8636
8637    /// A candidate rendered for judging, with the leak policy applied.
8638    fn view(&self, c: &Candidate) -> CandidateView {
8639        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8640            .unwrap_or_default();
8641        let (patch, _) = blind::sanitize_patch(
8642            &format!("candidate {} patch", c.label),
8643            &raw,
8644            &self.state.config.blind,
8645        );
8646        CandidateView {
8647            label: c.label,
8648            branch: c.branch.clone(),
8649            summary: c.summary.clone(),
8650            stat: c.stat.clone(),
8651            patch,
8652        }
8653    }
8654
8655    /// The full candidate set as prompt text, for seats with no live session.
8656    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8657        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8658        prompt::judge(
8659            "(see above)",
8660            &views,
8661            self.roles.judges.len(),
8662            base_short,
8663            "en",
8664        )
8665    }
8666
8667    /// The final-vote prompt with everything a seat that has no session of its
8668    /// own needs: the candidates, the seat's own ranking and reasons, and the
8669    /// anonymised deliberation it took part in (only when there was one, so a
8670    /// handed-over seat never sees more than the seat it replaces did). The
8671    /// `Final vote` heading stays first.
8672    fn vote_prompt_full(
8673        &self,
8674        j: usize,
8675        labels: &[char],
8676        language: &str,
8677        candidates: &[Candidate],
8678        base_short: &str,
8679    ) -> String {
8680        let mut text = format!(
8681            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8682            prompt::final_vote(labels, language),
8683            self.state.instruction,
8684            self.candidate_block(candidates, base_short)
8685        );
8686        if let Some(own) = self
8687            .state
8688            .judgements
8689            .get(j)
8690            .filter(|r| !r.ranking.is_empty())
8691        {
8692            let reasons = own
8693                .reasons
8694                .iter()
8695                .map(|(k, v)| format!("- {k}: {v}"))
8696                .collect::<Vec<_>>()
8697                .join("\n");
8698            text.push_str(&format!(
8699                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8700                own.ranking.iter().collect::<String>(),
8701                if reasons.is_empty() {
8702                    ""
8703                } else {
8704                    ", because:\n"
8705                }
8706            ));
8707        }
8708        if !self.state.deliberation.is_empty() {
8709            text.push_str("\n# What was argued before this vote\n");
8710            for t in self.transcript(&[], j) {
8711                text.push_str(&format!(
8712                    "\n## {}{}\n\n{}\n",
8713                    t.who,
8714                    if t.is_self { " (you)" } else { "" },
8715                    t.body.trim()
8716                ));
8717            }
8718        }
8719        text
8720    }
8721
8722    /// Anonymised transcript for judge `self_idx`.
8723    ///
8724    /// The initial rankings are always the opening statements. Seeding them
8725    /// only when no turn had been taken yet meant every judge after the first
8726    /// argued against a single voice instead of against the actual split — the
8727    /// disagreement is the information, so it is always on the table.
8728    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8729        let mut turns = Vec::new();
8730        for j in &self.state.judgements {
8731            if j.ranking.is_empty() {
8732                continue;
8733            }
8734            let reasons = j
8735                .reasons
8736                .iter()
8737                .map(|(k, v)| format!("- {k}: {v}"))
8738                .collect::<Vec<_>>()
8739                .join("\n");
8740            turns.push(Turn {
8741                who: format!("Judge {} (opening ranking)", j.judge),
8742                is_self: j.judge == self_idx + 1,
8743                body: format!(
8744                    "Ranked {}{}{reasons}",
8745                    j.ranking.iter().collect::<String>(),
8746                    if reasons.is_empty() {
8747                        ""
8748                    } else {
8749                        ", because:\n"
8750                    }
8751                ),
8752            });
8753        }
8754        for t in self
8755            .state
8756            .deliberation
8757            .iter()
8758            .flat_map(|r| r.turns.iter())
8759            .chain(current)
8760        {
8761            turns.push(Turn {
8762                who: format!("Judge {}", t.judge),
8763                is_self: t.judge == self_idx + 1,
8764                body: t.body.clone(),
8765            });
8766        }
8767        turns
8768    }
8769}
8770
8771/// Does this seat still hold the context a follow-up prompt would rely on?
8772fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8773    agent::has_session(spec.kind, seat, sessions)
8774}
8775
8776/// The next entry in `roster` after `start`, never wrapping back to the
8777/// front, whose id is not in `tried` yet.
8778///
8779/// Starts one past `start` rather than at the front of `roster`: `start` is
8780/// the seat's own original position, and a seat whose candidate slot already
8781/// sits on the roster's second entry must fall through to the third next, not
8782/// restart at the first — which is very likely a different candidate's own
8783/// agent already. Never wraps back past `start`, for the same reason: an
8784/// entry earlier in the roster than the seat's own position is almost
8785/// certainly some *other* candidate slot's own agent, and once the tail of
8786/// the roster is exhausted there are no more untried agents for *this* seat
8787/// to fall through to — the caller's fallback chain ends there, exactly as
8788/// "no further untried agents remain in the list for that seat" asks for.
8789///
8790/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8791/// the same id twice (an operator's `roles.implementers` typo, or a
8792/// `[[agents]]` list reused across roles) must not let
8793/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8794/// over `roster` either finds an untried id or runs out, so this always
8795/// terminates regardless of duplicates.
8796fn next_untried_in_roster<'a>(
8797    roster: &'a [AgentSpec],
8798    start: usize,
8799    tried: &BTreeSet<String>,
8800) -> Option<&'a AgentSpec> {
8801    roster
8802        .get(start + 1..)?
8803        .iter()
8804        .find(|s| !tried.contains(&s.id))
8805}
8806
8807/// The successor for a seat, shared by all four seat kinds (implement, judge,
8808/// review, advise). `others` holds the ids that *currently* occupy the other
8809/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8810/// sees them), so roster entries beyond the seat count act as spares: a failed
8811/// seat goes to an agent no other seat holds whenever the roster permits.
8812///
8813/// `carried` is `Some` only for the review loop's carried failure history.
8814/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8815/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8816/// whole roster: untried and not another seat's occupant; (3) the plain walk
8817/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8818/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8819/// a better outcome than an empty seat, and it keeps the answer to "is there
8820/// a successor at all" exactly what it was before occupants were considered,
8821/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8822/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8823/// carried failure is a cheaper bet than doubling an agent on the panel, and
8824/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8825/// the same round are handled one at a time, so a seat later in the batch
8826/// sees an earlier one's new occupant but not yet its own freed agent.
8827fn pick_successor<'a>(
8828    roster: &'a [AgentSpec],
8829    start: usize,
8830    tried: &BTreeSet<String>,
8831    carried: Option<&BTreeSet<String>>,
8832    others: &BTreeSet<String>,
8833) -> Option<&'a AgentSpec> {
8834    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8835    roster
8836        .get(start + 1..)
8837        .and_then(|tail| {
8838            tail.iter()
8839                .filter(free)
8840                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8841        })
8842        .or_else(|| {
8843            carried
8844                .is_some()
8845                .then(|| roster.iter().find(free))
8846                .flatten()
8847        })
8848        .or_else(|| match carried {
8849            Some(c) => next_for_seat(roster, start, tried, c),
8850            None => next_untried_in_roster(roster, start, tried),
8851        })
8852}
8853
8854/// The next agent for a seat that carries its failure history across rounds
8855/// (the review loop). `round_tried` is this round's own bound and starts
8856/// empty every round; `carried_failed` only decides priority.
8857///
8858/// First: an id walking forward from `start`, never wrapping, that is neither
8859/// tried this round nor failed in an earlier one. Only when that is exhausted
8860/// does it rescue: the first roster id (in roster order, so this one *does*
8861/// look before `start`) not yet tried this round, which is by then a carried
8862/// failure. Each id is rescued at most once per round, so it cannot loop.
8863fn next_for_seat<'a>(
8864    roster: &'a [AgentSpec],
8865    start: usize,
8866    round_tried: &BTreeSet<String>,
8867    carried_failed: &BTreeSet<String>,
8868) -> Option<&'a AgentSpec> {
8869    roster
8870        .get(start + 1..)?
8871        .iter()
8872        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8873        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8874}
8875
8876/// Where a reviewer seat starts a round: the agent that last answered it when
8877/// it is still on the roster and not marked failed, else the spec's own agent
8878/// unless it failed, else the next roster agent that has not failed, else the
8879/// spec's own agent again (the whole roster failed). Ids no longer on the
8880/// roster are ignored. An empty roster has no handover, so the spec stands.
8881fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8882    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8883        return spec;
8884    };
8885    let ok = |id: &str| !h.failed.contains(id);
8886    if let Some(last) = h.last_ok.as_deref()
8887        && ok(last)
8888        && let Some(s) = roster.iter().find(|s| s.id == last)
8889    {
8890        return s.clone();
8891    }
8892    if ok(&spec.id) {
8893        return spec;
8894    }
8895    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8896    roster
8897        .get(from + 1..)
8898        .into_iter()
8899        .flatten()
8900        .chain(roster.iter())
8901        .find(|s| ok(&s.id))
8902        .cloned()
8903        .unwrap_or(spec)
8904}
8905
8906/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
8907/// seed so a CLI that mints its session id up front (`--session-id`) never
8908/// reuses the uuid the previous agent already opened under the same seat key.
8909pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8910    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8911}
8912
8913/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
8914/// so the classifier below and the code that builds the message cannot drift.
8915const TIMED_OUT: &str = "timed out";
8916
8917impl FailClass {
8918    /// `None` for an answer; otherwise how the turn failed.
8919    fn of(out: &AgentOutcome) -> Option<Self> {
8920        match out {
8921            AgentOutcome::Ok(_) => None,
8922            AgentOutcome::Quota(_) => Some(Self::Quota),
8923            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8924            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8925            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8926            // Not a failure of the agent. Callers check for it first; `None`
8927            // here means "no class to hand over on".
8928            AgentOutcome::ResourceBlocked(_) => None,
8929        }
8930    }
8931
8932    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
8933    fn stem_word(&self) -> &'static str {
8934        match self {
8935            Self::Quota => "quota",
8936            _ => "handover",
8937        }
8938    }
8939}
8940
8941/// The message's first line with its variable parts removed — digit runs and
8942/// path-like tokens — so "exit code 2" and "exit code 7" read
8943/// as one kind of failure.
8944fn failure_signature(msg: &str) -> String {
8945    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8946    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
8947    // the historical machine key while the process diagnosis uses readable prose.
8948    if let Some(code) = line
8949        .strip_prefix("process exited with no usable output (exit code ")
8950        .and_then(|s| s.strip_suffix(')'))
8951        .and_then(|s| s.parse::<i32>().ok())
8952    {
8953        line = format!("exited with some({code}) and no usable output");
8954    } else if matches!(
8955        line.as_str(),
8956        "process exited with no usable output (no exit code reported)"
8957            | "process exited with no usable output (exit code unavailable)"
8958    ) {
8959        line = "exited with none and no usable output".to_owned();
8960    }
8961    let mut out = Vec::new();
8962    for word in line.split_whitespace() {
8963        if word.contains('/') || word.contains('\\') {
8964            out.push("<path>".to_owned());
8965            continue;
8966        }
8967        let mut w = String::new();
8968        let mut in_digits = false;
8969        for c in word.chars() {
8970            if c.is_ascii_digit() {
8971                if !in_digits {
8972                    w.push('#');
8973                }
8974                in_digits = true;
8975            } else {
8976                in_digits = false;
8977                w.push(c);
8978            }
8979        }
8980        out.push(w);
8981    }
8982    out.join(" ").chars().take(120).collect()
8983}
8984
8985/// Whether a seat that just failed with `cur` may go to the next roster agent.
8986/// A quota or a timeout always may. Any other failure may not when the agent
8987/// before it failed the same way: an error the prompt causes would otherwise
8988/// walk the whole roster. `prev` is the class of the immediately preceding
8989/// agent's failure, so a quota or timeout in between breaks the run of
8990/// identical failures by itself.
8991fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8992    match cur {
8993        FailClass::Quota | FailClass::Timeout => true,
8994        FailClass::Other(_) => prev != Some(cur),
8995    }
8996}
8997
8998/// A short human reason for a failed outcome, for the handover record.
8999fn fail_reason(out: &AgentOutcome) -> String {
9000    match out {
9001        AgentOutcome::Ok(_) => String::new(),
9002        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9003        AgentOutcome::Dropped(o) => format!(
9004            "the CLI dropped the stream ({})",
9005            o.dropped
9006                .as_ref()
9007                .map(|d| d.why.as_str())
9008                .unwrap_or("it ended without delivering its answer")
9009        ),
9010        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9011            e.lines().next().unwrap_or("").chars().take(160).collect()
9012        }
9013    }
9014}
9015
9016/// Note one handover in the run: the structured record and, in the timeline,
9017/// the sentence a person reads. A quota keeps the wording it always had.
9018pub(crate) fn record_handover(
9019    state: &mut RunState,
9020    node: &str,
9021    seat: &str,
9022    from: &str,
9023    to: &str,
9024    class: &FailClass,
9025    reason: &str,
9026) {
9027    let message = if *class == FailClass::Quota {
9028        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9029    } else {
9030        format!("{seat}: handed over {from} -> {to} ({reason})")
9031    };
9032    state.event(node, message);
9033    state.handovers.push(Handover {
9034        at: Timestamp::now(),
9035        node: node.to_owned(),
9036        seat: seat.to_owned(),
9037        from: from.to_owned(),
9038        to: to.to_owned(),
9039        reason: reason.to_owned(),
9040    });
9041}
9042
9043/// Did this reply report running a command whose own CLI never confirmed an
9044/// exit status?
9045///
9046/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9047/// command *finished* (see that type's own doc), so this can only be `true`
9048/// for a command whose completion event carried no readable exit code — not
9049/// for one that simply is not mentioned at all. That is the one signal this
9050/// crate can read, from the same record `state.jobs` renders, about a reply
9051/// standing next to work its own CLI cannot vouch for finishing; it is
9052/// deliberately not a check on the exit code's *value* (a fixer legitimately
9053/// runs a command that fails mid-iteration before it succeeds) and not a
9054/// guess at a command still running in the background (which emits no event
9055/// at all, and so leaves no evidence here to find).
9056fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9057    commands.iter().any(|c| c.exit_code.is_none())
9058}
9059
9060/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9061/// trusted as a verified no-op — the adoption guard's own text-level half.
9062///
9063/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9064/// timed out): a marker only earns the benefit of the doubt from a turn the
9065/// CLI itself vouches for finishing properly, the same house style
9066/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9067/// *fix* report to for `commands`. A candidate that timed out, exited
9068/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9069/// is, whatever prose it wrote — this returns `None` before it ever looks at
9070/// `text`. The remaining guards (the tree really is empty, the evidence is
9071/// non-empty) are the caller's: this only reads what the reply *claimed*.
9072fn verified_noop_claim(
9073    usable: bool,
9074    commands: &[agent::CommandEvidence],
9075    text: &str,
9076) -> Option<String> {
9077    (usable && !has_unconfirmed_command(commands))
9078        .then(|| verdict::verified_noop(text))
9079        .flatten()
9080}
9081
9082fn short(commit: &str) -> String {
9083    commit.chars().take(7).collect()
9084}
9085
9086fn make_executable(path: &Path) -> Result<()> {
9087    #[cfg(unix)]
9088    {
9089        use std::os::unix::fs::PermissionsExt as _;
9090        let mut perms = std::fs::metadata(path)?.permissions();
9091        perms.set_mode(0o755);
9092        std::fs::set_permissions(path, perms)?;
9093    }
9094    #[cfg(not(unix))]
9095    {
9096        let _ = path;
9097    }
9098    Ok(())
9099}
9100
9101/// What every seat in one batch shares: where the answers are attributed, the
9102/// prompt overlay they inherit, and the build cache they are told to use.
9103///
9104/// A struct rather than four more parameters: `wave` also needs the run's
9105/// state (to record who is answering right now) and the attempt number, and
9106/// eight positional arguments is both unreadable and a clippy error.
9107struct WaveCtx<'a> {
9108    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9109    /// paid for it.
9110    run: &'a str,
9111    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9112    node: &'a str,
9113    prompts: &'a Prompts,
9114    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9115    cache: Option<&'a Path>,
9116    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9117    /// `JobRecord::round`. `None` for every other node.
9118    round: Option<usize>,
9119    /// Carry each seat's failed-agent history across waves (the review loop
9120    /// only): start-of-round priority and handover choice read
9121    /// [`RunState::seat_history`], and every answer or failure writes it.
9122    carry_seats: bool,
9123    /// The run's park / interrupt requests, so a wait for a shared resource
9124    /// can be abandoned the moment the operator asks to park.
9125    pauses: Option<(&'a Pause, &'a Pause)>,
9126}
9127
9128/// Shared scheduling policy for operations and spawned seat invocations.
9129fn repository_wait(
9130    state: &RunState,
9131    pauses: Option<(&Pause, &Pause)>,
9132) -> crate::repo_lock::WaitPolicy {
9133    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9134    let inherited = crate::repo_lock::WaitPolicy::current();
9135    crate::repo_lock::WaitPolicy::new(
9136        Duration::from_secs(state.config.graph.timeout_resource_wait),
9137        move || {
9138            inherited.cancelled()
9139                || pauses
9140                    .as_ref()
9141                    .is_some_and(|(a, b)| a.parked() || b.parked())
9142        },
9143    )
9144}
9145
9146fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9147    if error.is::<crate::repo_lock::ResourceBlocked>() {
9148        state.resource_block = Some(crate::run::ResourceBlock {
9149            node: "repository".into(),
9150            kind: if error.to_string().contains("disk pressure") {
9151                "disk pressure"
9152            } else {
9153                "repository coordination"
9154            }
9155            .into(),
9156            detail: error.to_string(),
9157            at: Timestamp::now(),
9158        });
9159        state.event("resource", error.to_string());
9160    }
9161}
9162
9163async fn coordinate_operation(
9164    state: &mut RunState,
9165    cwd: &Path,
9166    wait: &crate::repo_lock::WaitPolicy,
9167) -> Result<Option<crate::repo_lock::SeatLease>> {
9168    let result =
9169        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9170            .await;
9171    if let Err(e) = &result {
9172        record_repository_block(state, e);
9173    }
9174    result
9175}
9176
9177/// Run one job, honouring the parallelism budget.
9178async fn run_one(
9179    job: SeatJob,
9180    sem: Arc<Semaphore>,
9181    ctx: &WaveCtx<'_>,
9182    state: &mut RunState,
9183    attempt: usize,
9184) -> (SeatState, AgentOutcome) {
9185    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9186        .await
9187        .pop()
9188        .expect("one job in, one result out");
9189    (seat, out)
9190}
9191
9192/// Run every job concurrently, capped by the semaphore, preserving order.
9193///
9194/// Every seat in the batch is recorded into [`RunState::active`] before the
9195/// wave starts and cleared as each answer lands, so the run's own record says
9196/// who is still being waited on rather than only who finished.
9197async fn wave(
9198    jobs: Vec<SeatJob>,
9199    sem: Arc<Semaphore>,
9200    ctx: &WaveCtx<'_>,
9201    state: &mut RunState,
9202    attempt: usize,
9203) -> Vec<(usize, SeatState, AgentOutcome)> {
9204    let WaveCtx {
9205        run,
9206        node,
9207        prompts,
9208        cache,
9209        round,
9210        carry_seats: _,
9211        pauses,
9212    } = *ctx;
9213    for job in &jobs {
9214        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9215    }
9216    if let Err(e) = state.save() {
9217        // A failed persist of "who is answering right now" must not abort the
9218        // wave: the seats are already being asked, and the alternative is
9219        // losing the answers to save a status line nobody may even be
9220        // watching.
9221        tracing::warn!("could not persist in-progress seats: {e:#}");
9222    }
9223    // Hold the shared build cache's lease for the whole batch, not per job:
9224    // several candidates (an implement wave) or a fixer legitimately share
9225    // one cache concurrently within this run, and that stays untouched — a
9226    // single lease taken once for the whole wave and released once it is
9227    // done is what stops a *different* borrower (another run's own wave, its
9228    // e2e/gate, a human's `magi review`) from interleaving a build into the
9229    // same directory while this one is in flight. Best-effort, not
9230    // all-or-nothing: a wave that cannot get the lease within its own
9231    // longest job's budget still runs — an hour of paid implementer calls is
9232    // not thrown away over cache contention — but every write-allowed seat
9233    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9234    // below), the same fallback a read-only seat always gets, rather than
9235    // building into a directory this run was never granted. The identity
9236    // record is still invalidated below either way, so the next tracked
9237    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9238    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9239    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9240    // never borrowed from a job's agent budget (and never charged to it either,
9241    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9242    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9243    let cache_guard = if let Some(cache_dir) = cache {
9244        if jobs_had_a_writer {
9245            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9246            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9247            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9248            {
9249                Ok(Some(g)) => Some(g),
9250                // Parking: leave without spawning anything. The run parks at
9251                // the next boundary and the seats are asked afresh on resume.
9252                Ok(None) => {
9253                    state.resource_block = Some(crate::run::ResourceBlock {
9254                        node: node.to_owned(),
9255                        kind: "shared build cache".to_owned(),
9256                        detail: "the run is parking".to_owned(),
9257                        at: Timestamp::now(),
9258                    });
9259                    return block_wave(
9260                        jobs,
9261                        state,
9262                        node,
9263                        attempt,
9264                        "parked while waiting for the shared build cache",
9265                    );
9266                }
9267                // An unreadable lease is not contention: keep the old
9268                // fallback (no shared cache for this wave's seats).
9269                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9270                Err(e) => {
9271                    // Writing seats may not build into a cache this run does
9272                    // not hold, and an agent that cannot be given one is not
9273                    // asked at all: the run stops resumably instead.
9274                    let why = format!("{e:#}");
9275                    state.resource_block = Some(crate::run::ResourceBlock {
9276                        node: node.to_owned(),
9277                        kind: "shared build cache".to_owned(),
9278                        detail: why.clone(),
9279                        at: Timestamp::now(),
9280                    });
9281                    state.event(
9282                        "resource",
9283                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9284                    );
9285                    return block_wave(jobs, state, node, attempt, &why);
9286                }
9287            }
9288        } else {
9289            None
9290        }
9291    } else {
9292        None
9293    };
9294    // Every job keeps its whole budget: the wait above was not the agent's.
9295    // Restart the seats' clocks so "started" means "being spawned now".
9296    state.seats_spawning(node, attempt);
9297    if let Err(e) = state.save() {
9298        tracing::warn!("could not persist the start of a wave: {e:#}");
9299    }
9300    // Queueing behind this wave's own seats for a parallelism slot is normal
9301    // (each of them is bounded by its own agent timeout); only a wait beyond
9302    // all of them plus the resource limit means the slots never came back.
9303    let slot_limit = resource_limit
9304        + jobs
9305            .iter()
9306            .map(|j| j.timeout)
9307            .max()
9308            .unwrap_or_default()
9309            .saturating_mul(jobs.len() as u32);
9310    let mut set = tokio::task::JoinSet::new();
9311    // Seats announce themselves here the moment they hold a parallelism
9312    // permit, i.e. just before their CLI is spawned.
9313    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9314    let overlay = prompts.overlay(node);
9315    let repository_wait = repository_wait(state, pauses);
9316    for (i, mut job) in jobs.into_iter().enumerate() {
9317        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9318        if cache.is_some() {
9319            job.prompt.push('\n');
9320            job.prompt
9321                .push_str(&prompt::build_cache_note(node, job.allow_write));
9322        }
9323        let sem = Arc::clone(&sem);
9324        let run = run.to_owned();
9325        let node = node.to_owned();
9326        // Only implementers were told about the task's attachments, so only
9327        // their seats get the directory widened for reading.
9328        let attachments = if node == "implement" {
9329            state.attachments.clone()
9330        } else {
9331            Vec::new()
9332        };
9333        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9334        // `prompt::build_cache_note`'s doc for why setting it anyway is
9335        // exactly how a sandboxed reviewer's write refusal got reported as a
9336        // defect in the patch, not a property of its own seat. And a
9337        // write-allowed one is handed it only when the lease above was
9338        // actually acquired: a wave that could not get it (`cache_guard` is
9339        // `None`, see its own comment) must not send seats to build into a
9340        // directory this run does not hold - that is the exact concurrent,
9341        // unmanaged-write race this module exists to prevent, not something
9342        // "proceeding anyway" is allowed to reintroduce.
9343        let cache = cache
9344            .filter(|_| job.allow_write && cache_guard.is_some())
9345            .map(Path::to_path_buf);
9346        let spawned_tx = spawned_tx.clone();
9347        let repository_wait = repository_wait.clone();
9348        let coordination = state.config.coordination.clone();
9349        let fetched_base = state.fetched_base.clone();
9350        set.spawn(async move {
9351            // A wait for a parallelism slot is scheduling, not the agent: it
9352            // is bounded by its own limit and ends as a resource block, never
9353            // as a timeout of the agent that was not started.
9354            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9355                Ok(p) => p,
9356                Err(_) => {
9357                    return (
9358                        i,
9359                        job.seat,
9360                        AgentOutcome::ResourceBlocked(format!(
9361                            "no parallel slot became free within {}s",
9362                            slot_limit.as_secs()
9363                        )),
9364                        false,
9365                    );
9366                }
9367            };
9368            spawned_tx.send(job.seat.key.clone()).ok();
9369            let mut seat = job.seat;
9370            let out = repository_wait
9371                .scope(agent::invoke(
9372                    &job.spec,
9373                    &mut seat,
9374                    &Invocation {
9375                        scope: crate::agent::InvocationScope::Repository,
9376                        coordination: coordination.as_ref(),
9377                        fetched_base: fetched_base.as_ref(),
9378                        cwd: &job.cwd,
9379                        prompt: &job.prompt,
9380                        timeout: job.timeout,
9381                        allow_write: job.allow_write,
9382                        unsandboxed: false,
9383                        sessions: job.sessions,
9384                        artifacts: &job.artifacts,
9385                        stem: &job.stem,
9386                        run: &run,
9387                        node: &node,
9388                        cache_dir: cache.as_deref(),
9389                        attachments: &attachments,
9390                        writable: &[],
9391                    },
9392                ))
9393                .await;
9394            let interrupted = out
9395                .as_ref()
9396                .err()
9397                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9398            let out = match out {
9399                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9400                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9401                // Billed work the CLI failed to hand over is not an ordinary
9402                // failure, but its text is the CLI's raw error JSON, not an
9403                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9404                // read it as one by forgetting to check. `usable()` is always
9405                // false here (dropped implies an empty response), so this has
9406                // to be checked before the catch-all `Failed` below or the
9407                // one shape this exists for is lost with the rest.
9408                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9409                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9410                Ok(o) => AgentOutcome::Failed(format!(
9411                    "process exited with no usable output ({})",
9412                    crate::notices::exit_status(o.exit_code)
9413                )),
9414                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9415                    AgentOutcome::ResourceBlocked(e.to_string())
9416                }
9417                Err(e) => AgentOutcome::Failed(e.to_string()),
9418            };
9419            (i, seat, out, interrupted)
9420        });
9421    }
9422    drop(spawned_tx);
9423    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9424    loop {
9425        let joined = tokio::select! {
9426            Some(key) = spawned_rx.recv() => {
9427                // The clock of this seat starts now, not when it queued.
9428                state.seat_spawned(&key);
9429                if let Err(e) = state.save() {
9430                    tracing::warn!("could not persist a seat's start: {e:#}");
9431                }
9432                continue;
9433            }
9434            joined = set.join_next() => joined,
9435        };
9436        let Some(joined) = joined else {
9437            break;
9438        };
9439        let (i, seat, out, interrupted) = match joined {
9440            Ok(v) => v,
9441            // No seat to clear: a panicked task never reported which one it
9442            // was. The defensive sweep below this loop is what stops that
9443            // seat's `active` entry from surviving forever.
9444            Err(e) => {
9445                tracing::error!("agent task panicked: {e}");
9446                continue;
9447            }
9448        };
9449        state.seat_finished(&seat.key);
9450        if interrupted
9451            && node == "implement"
9452            && let Some(label) = state
9453                .candidates
9454                .iter()
9455                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9456                .map(|candidate| candidate.label)
9457        {
9458            state.candidate_preservation.insert(
9459                label,
9460                "interrupted implementation awaiting persistence".into(),
9461            );
9462            state.interrupted_implementations.insert(label);
9463        }
9464        if let AgentOutcome::ResourceBlocked(why) = &out
9465            && state.resource_block.is_none()
9466        {
9467            state.resource_block = Some(crate::run::ResourceBlock {
9468                node: node.to_owned(),
9469                kind: if why.contains("disk pressure") {
9470                    "disk pressure"
9471                } else if why.contains("repository coordination") {
9472                    "repository coordination"
9473                } else {
9474                    "parallel slot"
9475                }
9476                .to_owned(),
9477                detail: why.clone(),
9478                at: Timestamp::now(),
9479            });
9480            let activity = if interrupted {
9481                "agent execution interrupted"
9482            } else {
9483                "no agent was started"
9484            };
9485            state.event("resource", format!("{node}: {activity} - {why}"));
9486        }
9487        record_jobs(state, node, round, &seat.key, &out);
9488        if let Err(e) = state.save() {
9489            tracing::warn!("could not persist a seat's completion: {e:#}");
9490        }
9491        if collected.len() <= i {
9492            collected.resize_with(i + 1, || None);
9493        }
9494        collected[i] = Some((i, seat, out));
9495    }
9496    // Belt-and-braces for the panic branch above: every seat this exact batch
9497    // started shares this `(node, attempt)` pair, and every seat that finished
9498    // normally already cleared itself, so anything left tagged with it here
9499    // can only be a panicked task's leftover. Cleared unconditionally rather
9500    // than left to read as still answering forever.
9501    if state
9502        .active
9503        .values()
9504        .any(|a| a.node == node && a.attempt == attempt)
9505    {
9506        state
9507            .active
9508            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9509        if let Err(e) = state.save() {
9510            tracing::warn!("could not persist the end of a wave: {e:#}");
9511        }
9512    }
9513    // Whether or not the lease above was actually held, several worktrees
9514    // may just have built into the cache with nothing here able to name one
9515    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9516    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9517    // clean it might not have strictly needed; trusting a stale match would
9518    // cost it a wrong answer.
9519    if let Some(cache_dir) = cache
9520        && jobs_had_a_writer
9521    {
9522        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9523    }
9524    if let Some(guard) = cache_guard {
9525        guard.release();
9526    }
9527    collected.into_iter().flatten().collect()
9528}
9529
9530/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9531/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9532/// this is data collection, not the fix-specific completion contract in
9533/// [`Runner::continue_fix_report`], and applies regardless of which node
9534/// asked.
9535///
9536/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9537/// evidence from; `Failed` does not, and correctly contributes nothing — a
9538/// timeout or crash is not itself evidence about a command the seat may have
9539/// started.
9540fn record_jobs(
9541    state: &mut RunState,
9542    node: &str,
9543    round: Option<usize>,
9544    seat: &str,
9545    out: &AgentOutcome,
9546) {
9547    let commands: &[agent::CommandEvidence] = match out {
9548        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9549        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9550    };
9551    let checked_at = Timestamp::now();
9552    for c in commands {
9553        state.jobs.push(JobRecord {
9554            node: node.to_owned(),
9555            round,
9556            seat: seat.to_owned(),
9557            id: c.id.clone(),
9558            description: c.description.clone(),
9559            checked_at,
9560            status: match c.exit_code {
9561                Some(0) => JobStatus::Completed,
9562                Some(_) => JobStatus::Failed,
9563                None => JobStatus::Unknown,
9564            },
9565            exit_code: c.exit_code,
9566            result_summary: c.result_summary.clone(),
9567            source: c.source.clone(),
9568        });
9569    }
9570}
9571
9572/// Is a review round clean, given how many reviewer seats answered against
9573/// how many the round expected?
9574///
9575/// A seat that never answered (timeout, crash, unparsable output) is not a
9576/// seat that read the patch and found nothing — treating it as such is
9577/// exactly the bug this function exists to close. Under the default `block`
9578/// policy a missing seat can never be clean; `warn` still requires the seats
9579/// that *did* answer to have found nothing blocking and verification to be
9580/// green.
9581///
9582/// `quota_missing` narrows that `block` default for exactly one cause of
9583/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9584/// a session limit lifts by the very next round buys nothing — the seat is
9585/// asked again with the same quota — so once every missing seat is accounted
9586/// for by a quota loss (and at least one seat *did* answer, so a decision has
9587/// something to rest on) the round is decided on the panel that could answer,
9588/// same as `warn` would. A panel that lost every seat to quota is not
9589/// decided here: `answered == 0` falls through to the existing `block`
9590/// fallback so a fully collapsed panel still waits rather than landing on no
9591/// review at all.
9592fn round_is_clean(
9593    blocking: usize,
9594    e2e_ok: bool,
9595    answered: usize,
9596    expected: usize,
9597    quota_missing: usize,
9598    policy: IncompleteReviewPolicy,
9599) -> bool {
9600    if blocking != 0 || !e2e_ok {
9601        return false;
9602    }
9603    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9604        return true;
9605    }
9606    answered > 0 && expected - answered <= quota_missing
9607}
9608
9609/// The review loop's own conclusion, derived entirely from its persisted
9610/// round records and the round budget that produced them — never from
9611/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9612/// recomputes the identical answer regardless of what an earlier node in the
9613/// same walk, or a previous walk, did to `status`.
9614///
9615/// `None` while more rounds remain to try, including when review never ran
9616/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9617/// gone clean, or the budget is spent, or the tree has stopped moving (see
9618/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9619///
9620/// - An incomplete panel that raised nothing is missing input, not a
9621///   verified tree — never a hand-off candidate, whatever verification said
9622///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9623/// - Otherwise, green e2e on the last round hands off (see
9624///   [`Runner::stop_reviewing`]); red e2e blocks.
9625///
9626/// A last round whose own verification is still `ResourceBlocked` — magi
9627/// itself never got a command to run, not evidence the patch is broken —
9628/// is neither: this returns `None` for it too, the same as "more rounds
9629/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9630/// handling of that shape) instead of this cheap recomputation guessing a
9631/// verdict a real attempt never produced.
9632pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9633    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9634        return Some(RunStatus::Gating);
9635    }
9636    let last = reviews.last()?;
9637    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9638    if reviews.len() < max_rounds && !stagnant {
9639        return None;
9640    }
9641    if last.incomplete() && last.blocking == 0 {
9642        return Some(RunStatus::Blocked);
9643    }
9644    if last.e2e_status() == E2eStatus::ResourceBlocked {
9645        return None;
9646    }
9647    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9648        RunStatus::Gating
9649    } else {
9650        RunStatus::Blocked
9651    })
9652}
9653
9654/// How long a re-ask may take, given the budget the first attempt had.
9655///
9656/// A `nudged` retry is a request to restate an answer the seat has already
9657/// worked out: it carries no new work, so it does not deserve the original
9658/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9659/// 133 seconds while a third sat for over ten minutes on a resumed session
9660/// holding 410 KB of prior output - and because the retry had inherited the
9661/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9662/// judging round whose other seats were long finished.
9663///
9664/// A quarter of the budget, with a floor so that a deliberately short timeout
9665/// does not collapse to nothing. A retry that re-sends the whole prompt
9666/// (because the seat kept no context) is the original job again, and keeps the
9667/// original budget.
9668pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9669    if nudged {
9670        (full / 4).max(Duration::from_secs(120)).min(full)
9671    } else {
9672        full
9673    }
9674}
9675
9676/// Run a wave and parse each reply, re-asking the seats whose reply was
9677/// unusable.
9678///
9679/// The re-ask is a nudge rather than the whole prompt again when the seat still
9680/// holds its conversation, which is the difference between a cheap retry and
9681/// paying for the entire candidate set twice.
9682///
9683/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9684/// successor in `roster` is handed to it instead of being re-asked: the
9685/// handover is the retry. A seat with no successor left (a single-agent
9686/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9687/// the asks to one seat in one node number at most
9688/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9689/// is asked once (a dropped stream is nudged first), and only the last agent
9690/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9691/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9692/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9693/// other failure stops the chain when the previous agent failed the same way
9694/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9695/// [`has_context`] is false and the job's own full prompt and full budget are
9696/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9697/// intermediate ones are not losses) and is returned as a failure like any
9698/// other absent seat — the caller decides whether the panel still has a
9699/// quorum. An empty `roster` disables handover: failures are nudged as they
9700/// always were, and a quota is simply lost. A reply that fails to parse or
9701/// validate is the prompt's doing and is only ever nudged, never handed over.
9702///
9703/// The returned [`SeatState`] names the agent that answered (or tried last).
9704#[allow(clippy::too_many_arguments)]
9705async fn ask_json_wave<T>(
9706    jobs: Vec<SeatJob>,
9707    sem: Arc<Semaphore>,
9708    retries: usize,
9709    roster: &[AgentSpec],
9710    ctx: &WaveCtx<'_>,
9711    losses: &mut Vec<QuotaLoss>,
9712    state: &mut RunState,
9713    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9714) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9715where
9716    T: serde::de::DeserializeOwned + Send + 'static,
9717{
9718    ask_wave_with(
9719        jobs,
9720        sem,
9721        retries,
9722        roster,
9723        ctx,
9724        losses,
9725        state,
9726        &|text: &str| {
9727            let v = verdict::extract_json::<T>(text)?;
9728            validate(&v)?;
9729            Ok(v)
9730        },
9731    )
9732    .await
9733}
9734
9735/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9736/// a node whose answer is prose (deliberation) shares the same handover,
9737/// failure classification, quota bookkeeping and bounds instead of a copy.
9738///
9739/// A seat handed to another roster agent is sent the job's `handover` prompt
9740/// (when it has one) rather than `prompt`: the new agent has no session, so a
9741/// resume-style prompt would be incomplete. That holds for the handover ask
9742/// and for every nudge to that agent whose `has_context` is false.
9743#[allow(clippy::too_many_arguments)]
9744async fn ask_wave_with<T>(
9745    jobs: Vec<SeatJob>,
9746    sem: Arc<Semaphore>,
9747    retries: usize,
9748    roster: &[AgentSpec],
9749    ctx: &WaveCtx<'_>,
9750    losses: &mut Vec<QuotaLoss>,
9751    state: &mut RunState,
9752    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9753) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9754where
9755    T: Send + 'static,
9756{
9757    let n = jobs.len();
9758    let originals: Vec<SeatJob> = jobs;
9759    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9760    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9761    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9762    // answer, N once it has gone through N nudges. Read back once this
9763    // returns, so a caller building a history record (`ReviewRecord`) can
9764    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9765    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9766    // field's own doc.
9767    let mut nudges: Vec<usize> = vec![0; n];
9768    // The agent now occupying each seat, the ids it has already been through,
9769    // where in the roster the walk began, the class of the last failure, and
9770    // the stem word of a handover not yet asked (full prompt, full budget).
9771    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9772    let mut tried: Vec<BTreeSet<String>> = specs
9773        .iter()
9774        .map(|s| BTreeSet::from([s.id.clone()]))
9775        .collect();
9776    let starts: Vec<usize> = specs
9777        .iter()
9778        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9779        .collect();
9780    let carry = ctx.carry_seats && !roster.is_empty();
9781    // Carried across rounds: ids that failed the seat earlier, and how the
9782    // last failure went (so a repeat of it is not handed over again).
9783    let carried: Vec<BTreeSet<String>> = originals
9784        .iter()
9785        .map(|j| {
9786            state
9787                .seat_history
9788                .get(&j.seat.key)
9789                .filter(|_| carry)
9790                .map(|h| h.failed.clone())
9791                .unwrap_or_default()
9792        })
9793        .collect();
9794    let mut prev: Vec<Option<FailClass>> = originals
9795        .iter()
9796        .map(|j| {
9797            state
9798                .seat_history
9799                .get(&j.seat.key)
9800                .filter(|_| carry)
9801                .and_then(|h| h.last_fail.clone())
9802        })
9803        .collect();
9804    let next_agent =
9805        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9806            let others: BTreeSet<String> = specs
9807                .iter()
9808                .enumerate()
9809                .filter(|(j, _)| *j != i)
9810                .map(|(_, s)| s.id.clone())
9811                .collect();
9812            pick_successor(
9813                roster,
9814                starts[i],
9815                tried,
9816                carry.then(|| &carried[i]),
9817                &others,
9818            )
9819            .cloned()
9820        };
9821    let mut fresh: Vec<Option<String>> = vec![None; n];
9822    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9823    let mut pending: Vec<usize> = (0..n).collect();
9824
9825    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
9826    // asks: an agent with a successor is asked once and handed over, and only
9827    // a seat with no successor spends `retries` nudges on the same agent. This
9828    // only guarantees the loop's own termination whatever those say.
9829    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9830    for round in 0..max_rounds {
9831        if pending.is_empty() {
9832            break;
9833        }
9834        let mut batch = Vec::with_capacity(pending.len());
9835        let mut renudged: Vec<&str> = Vec::new();
9836        for &i in &pending {
9837            let src = &originals[i];
9838            // A seat now held by another agent than the job named has no
9839            // session of its own: it gets the full-context prompt whenever
9840            // it is asked in full (the handover ask, a nudge it cannot
9841            // resume).
9842            let full: &str = match &src.handover {
9843                Some(h) if specs[i].id != src.spec.id => h,
9844                _ => &src.prompt,
9845            };
9846            // The prompt and the budget are one decision: a nudge restates
9847            // finished work, a re-sent prompt redoes it.
9848            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9849                (
9850                    full.to_owned(),
9851                    src.timeout,
9852                    format!("{}-{word}-{}", src.stem, specs[i].id),
9853                )
9854            } else if nudges[i] == 0 {
9855                (full.to_owned(), src.timeout, src.stem.clone())
9856            } else {
9857                renudged.push(src.seat.key.as_str());
9858                let why = done[i]
9859                    .as_ref()
9860                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
9861                    .unwrap_or_else(|| "no parsable answer".to_owned());
9862                let nudge = prompt::nudge(&why);
9863                let nudged = has_context(&specs[i], &seats[i], src.sessions);
9864                let prompt = if nudged {
9865                    nudge
9866                } else {
9867                    format!("{full}\n\n---\n\n{nudge}")
9868                };
9869                (
9870                    prompt,
9871                    retry_budget(src.timeout, nudged),
9872                    format!("{}-retry{}", src.stem, nudges[i]),
9873                )
9874            };
9875            batch.push(SeatJob {
9876                spec: specs[i].clone(),
9877                seat: seats[i].clone(),
9878                cwd: src.cwd.clone(),
9879                prompt,
9880                timeout,
9881                allow_write: src.allow_write,
9882                sessions: src.sessions,
9883                artifacts: src.artifacts.clone(),
9884                stem,
9885                handover: None,
9886            });
9887        }
9888
9889        if !renudged.is_empty() {
9890            state.event(
9891                ctx.node,
9892                format!("retry {round}: re-asking {}", renudged.join(", ")),
9893            );
9894        }
9895        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9896        let mut still = Vec::new();
9897        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9898            seats[i] = seat;
9899            // No agent was asked: nothing to hand over, nudge, remember or
9900            // charge. The caller sees `resource_block` and stops the node.
9901            if let AgentOutcome::ResourceBlocked(why) = &out {
9902                done[i] = Some(Err(anyhow::anyhow!("{why}")));
9903                continue;
9904            }
9905            let class = FailClass::of(&out);
9906            // A dropped stream is nudged first (the conversation is still
9907            // there to pick up); only a seat whose nudges are spent hands over.
9908            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9909                && nudges[i] < retries
9910                && !roster.is_empty();
9911            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9912                let next = should_hand_over(prev[i].as_ref(), &cur)
9913                    .then(|| next_agent(i, &tried[i], &specs))
9914                    .flatten();
9915                if carry {
9916                    let h = state
9917                        .seat_history
9918                        .entry(originals[i].seat.key.clone())
9919                        .or_default();
9920                    h.failed.insert(specs[i].id.clone());
9921                    h.last_fail = Some(cur.clone());
9922                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9923                        h.last_ok = None;
9924                    }
9925                    // Saved before the next agent is asked, so a restart in
9926                    // between does not forget who failed.
9927                    if let Err(e) = state.save() {
9928                        tracing::warn!("could not persist a seat's failure history: {e:#}");
9929                    }
9930                }
9931                if let Some(next) = next {
9932                    record_handover(
9933                        state,
9934                        ctx.node,
9935                        &originals[i].seat.key,
9936                        &specs[i].id,
9937                        &next.id,
9938                        &cur,
9939                        &fail_reason(&out),
9940                    );
9941                    tried[i].insert(next.id.clone());
9942                    prev[i] = Some(cur.clone());
9943                    seats[i] =
9944                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9945                    specs[i] = next;
9946                    fresh[i] = Some(cur.stem_word().to_owned());
9947                    nudges[i] = 0;
9948                    done[i] = Some(Err(anyhow::anyhow!(
9949                        "handed over after: {}",
9950                        fail_reason(&out)
9951                    )));
9952                    still.push(i);
9953                    continue;
9954                }
9955            }
9956            if carry
9957                && !nudge_first
9958                && let Some(cur) = class.clone()
9959            {
9960                // The chain ended here (no successor, or a repeated failure).
9961                let h = state
9962                    .seat_history
9963                    .entry(originals[i].seat.key.clone())
9964                    .or_default();
9965                h.failed.insert(specs[i].id.clone());
9966                h.last_fail = Some(cur);
9967            }
9968            let parsed = match out {
9969                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9970                AgentOutcome::Quota(o) => {
9971                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9972                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9973                }
9974                // Not a parseable answer: the nudge loop re-asks it, which is
9975                // exactly what a dropped stream needs. Just don't hand its raw
9976                // error JSON to `extract_json`.
9977                AgentOutcome::Dropped(o) => {
9978                    let why = o
9979                        .dropped
9980                        .as_ref()
9981                        .map(|d| d.why.as_str())
9982                        .unwrap_or("the CLI ended the stream without delivering its answer");
9983                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9984                }
9985                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9986                    Err(anyhow::anyhow!(e))
9987                }
9988            };
9989            let quota = class == Some(FailClass::Quota);
9990            let failed = parsed.is_err();
9991            done[i] = Some(parsed);
9992            if carry && !failed {
9993                let h = state
9994                    .seat_history
9995                    .entry(originals[i].seat.key.clone())
9996                    .or_default();
9997                h.failed.remove(&specs[i].id);
9998                h.last_ok = Some(specs[i].id.clone());
9999                h.last_fail = None;
10000            }
10001            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10002            // fail the same way; and never re-ask a seat that already parsed.
10003            // A failed agent that still has a successor is not re-asked
10004            // either: the handover was its remedy and has just been refused
10005            // (the chain stops on a repeated failure class). A seat with no
10006            // successor left (a single-agent roster, the roster's tail, or an
10007            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10008            let agent_failure = class.is_some()
10009                && !nudge_first
10010                && !roster.is_empty()
10011                && next_agent(i, &tried[i], &specs).is_some();
10012            if failed && !quota && !agent_failure && nudges[i] < retries {
10013                nudges[i] += 1;
10014                still.push(i);
10015            }
10016        }
10017        pending = still;
10018    }
10019
10020    // One loss per seat whose chain ended on a quota: the intermediate ones
10021    // were absorbed by a handover and are not losses.
10022    for (i, q) in last_quota.into_iter().enumerate() {
10023        if let Some(reset) = q {
10024            losses.push(QuotaLoss {
10025                seat: originals[i].seat.key.clone(),
10026                node: ctx.node.to_owned(),
10027                at: Timestamp::now(),
10028                reset,
10029            });
10030        }
10031    }
10032
10033    seats
10034        .into_iter()
10035        .zip(done)
10036        .zip(nudges)
10037        .map(|((seat, res), attempts)| {
10038            (
10039                seat,
10040                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10041                attempts,
10042            )
10043        })
10044        .collect()
10045}
10046
10047/// Did the wave end without asking any agent (see [`block_wave`])?
10048fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10049    results
10050        .iter()
10051        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10052}
10053
10054/// Settle a wave that never spawned anything: every job answers
10055/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10056/// else about the seat (session, history, quota) is touched.
10057fn block_wave(
10058    jobs: Vec<SeatJob>,
10059    state: &mut RunState,
10060    node: &str,
10061    attempt: usize,
10062    why: &str,
10063) -> Vec<(usize, SeatState, AgentOutcome)> {
10064    state.resource_wait = None;
10065    state
10066        .active
10067        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10068    if let Err(e) = state.save() {
10069        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10070    }
10071    jobs.into_iter()
10072        .enumerate()
10073        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10074        .collect()
10075}
10076
10077/// Acquire the shared build cache's lease, waiting out contention within
10078/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10079/// unbounded wait is never acceptable).
10080///
10081/// A first, non-blocking check happens before ever waiting; if it finds the
10082/// lease busy, that fact is logged as a `verify` event *and* flushed with
10083/// [`RunState::save`] immediately — not only once the wait finally succeeds
10084/// or gives up — so a `magi show` run by a different process while this one
10085/// is still waiting reads a `run.json` that says so, rather than whatever it
10086/// looked like before the wait started. The same applies to the terminal
10087/// failure: logged and saved before this returns `Err`, so a caller that
10088/// could not get the lease at all still leaves a legible record of why.
10089async fn acquire_cache_lease(
10090    state: &mut RunState,
10091    cache_dir: &Path,
10092    owner: &crate::cache::Owner,
10093    budget: Duration,
10094    context: &str,
10095    cancel: &(dyn Fn() -> bool + Sync),
10096) -> Result<Option<crate::cache::Guard>> {
10097    let home = crate::run::home();
10098    let started = Instant::now();
10099    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10100        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10101        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10102        Err(e) => {
10103            state.event(
10104                "verify",
10105                format!("{context}: could not check the shared build cache: {e:#}"),
10106            );
10107            if let Err(e2) = state.save() {
10108                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10109            }
10110            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10111        }
10112    };
10113    state.event(
10114        "verify",
10115        format!(
10116            "{context}: waiting for the shared build cache at {} ({})",
10117            cache_dir.display(),
10118            busy.describe()
10119        ),
10120    );
10121    // Seats of this node are waiting, not running: say so, so the countdown a
10122    // reader sees only ever measures an agent that has been spawned.
10123    state.resource_wait = Some(crate::run::ResourceWait {
10124        node: context.to_owned(),
10125        kind: "shared build cache".to_owned(),
10126        holder: busy.describe(),
10127        since: Timestamp::now(),
10128        limit_secs: budget.as_secs(),
10129    });
10130    if let Err(e) = state.save() {
10131        tracing::warn!("could not persist a cache wait: {e:#}");
10132    }
10133    let remaining = budget.saturating_sub(started.elapsed());
10134    let waited = crate::cache::wait_for_unless(
10135        &home,
10136        cache_dir,
10137        owner,
10138        remaining,
10139        Duration::from_secs(5),
10140        cancel,
10141    )
10142    .await;
10143    state.resource_wait = None;
10144    match waited {
10145        Ok(g) => Ok(g),
10146        Err(e) => {
10147            state.event("verify", format!("{context}: {e:#}"));
10148            if let Err(e2) = state.save() {
10149                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10150            }
10151            Err(e)
10152        }
10153    }
10154}
10155
10156/// The lease state could not be read at all (unreadable lease file, a
10157/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10158/// known to be contended, so a wave degrades to running without a shared cache
10159/// instead of stopping on a hold that [`crate::triage`] could never release.
10160#[derive(Debug)]
10161struct CacheCheckError(String);
10162
10163impl std::fmt::Display for CacheCheckError {
10164    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10165        f.write_str(&self.0)
10166    }
10167}
10168
10169impl std::error::Error for CacheCheckError {}
10170
10171/// Run `body` — a verify command batch — while holding the shared build
10172/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10173/// never interleave with another borrower's build against the same
10174/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10175/// timeout, or a human's own `magi review`. See the `cache` module doc for
10176/// why this matters more than Cargo's own per-target locking covers — two
10177/// *different* worktrees building the same package name/version into one
10178/// cache directory is a staleness bug, not a lock contention one.
10179///
10180/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10181/// agent waves do. Repository scheduling also has its own bounded,
10182/// cancellable resource-wait policy. Contention must not consume the
10183/// verification budget: `body` receives the full `budget` after both leases
10184/// are acquired and the cache is fresh. Without a shared cache, only
10185/// repository coordination is acquired before handing over the full budget.
10186///
10187/// A lease that cannot be acquired within the resource wait is reported as a single
10188/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10189/// verification — the same shape a spawn failure already takes in
10190/// [`run_commands`], so a caller need not special-case it.
10191#[allow(clippy::too_many_arguments)]
10192async fn with_cache_lease<'s, F, Fut>(
10193    state: &'s mut RunState,
10194    cache_dir: Option<&Path>,
10195    node: &str,
10196    seat: &str,
10197    worktree: &Path,
10198    head: &str,
10199    budget: Duration,
10200    context: &str,
10201    wait: &crate::repo_lock::WaitPolicy,
10202    body: F,
10203) -> Result<(Vec<CommandOutcome>, bool)>
10204where
10205    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10206    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10207{
10208    // Match editing waves: cache first, repository second. In particular,
10209    // pre_gate must never hold the repository while waiting for a wave's cache.
10210    let started = Instant::now();
10211    let home = crate::run::home();
10212    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10213    let guard = if let Some(cache_dir) = cache_dir {
10214        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10215        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10216            wait.cancelled()
10217        })
10218        .await
10219        .and_then(|g| {
10220            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10221        }) {
10222            Ok(g) => Some(g),
10223            Err(e) => {
10224                return Ok((
10225                    vec![CommandOutcome {
10226                        command: "(waiting for the shared build cache)".into(),
10227                        code: None,
10228                        output_tail: e.to_string(),
10229                        duration_ms: started.elapsed().as_millis() as u64,
10230                        resource_blocked: true,
10231                    }],
10232                    false,
10233                ));
10234            }
10235        }
10236    } else {
10237        None
10238    };
10239    let _coordination = coordinate_operation(state, worktree, wait).await?;
10240    if let Some(cache_dir) = cache_dir {
10241        let identity = crate::cache::Identity::new(worktree, head);
10242        if let Err(e) = crate::cache::ensure_fresh(
10243            &home,
10244            cache_dir,
10245            &identity,
10246            &state.config.verify.cargo_manifests(),
10247        ) {
10248            state.event(
10249                "verify",
10250                format!(
10251                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10252                    worktree.display(),
10253                    short(head)
10254                ),
10255            );
10256            if let Some(guard) = guard {
10257                guard.release();
10258            }
10259            return Ok((
10260                vec![CommandOutcome {
10261                    command: "(confirming the shared build cache is fresh)".into(),
10262                    code: None,
10263                    output_tail: e.to_string(),
10264                    duration_ms: started.elapsed().as_millis() as u64,
10265                    resource_blocked: true,
10266                }],
10267                false,
10268            ));
10269        }
10270    }
10271    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10272    if !timed_out_pids.is_empty() {
10273        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10274    }
10275    // Drop repository coordination before releasing the cache, reversing the
10276    // acquisition order. A cancelled/error future also drops both guards.
10277    drop(_coordination);
10278    if let Some(guard) = guard {
10279        guard.release();
10280    }
10281    Ok((outcomes, retried))
10282}
10283
10284/// Poll `pids` — commands [`run_commands`] reports as still running when its
10285/// own timeout elapsed — until every one is confirmed gone, or
10286/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10287///
10288/// Real confirmation where confirmation is possible, not a substitute for
10289/// full process-tree observation: a grandchild the timed-out process spawned
10290/// and that survives independently of it is invisible to a pid check the
10291/// same way it always was, and continuing to observe and collect *that*
10292/// stays a different piece of work with its own owner. This only narrows a
10293/// fixed blind wait into an actual check of the pids this process does know
10294/// about.
10295async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10296    wait_for_pids_with(
10297        pids,
10298        crate::proc::pid_alive,
10299        LEASE_RELEASE_POLL,
10300        LEASE_RELEASE_MAX_WAIT,
10301    )
10302    .await;
10303}
10304
10305/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10306/// interval and ceiling supplied by the caller, so the polling *logic* -
10307/// returns as soon as every pid reports dead, gives up at the ceiling
10308/// otherwise - is testable on millisecond durations without asking the real
10309/// OS about a pid at all.
10310async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10311    pids: &[u32],
10312    alive: F,
10313    poll: Duration,
10314    max_wait: Duration,
10315) {
10316    let deadline = Instant::now() + max_wait;
10317    loop {
10318        if pids.iter().all(|&pid| !alive(pid)) {
10319            return;
10320        }
10321        if Instant::now() >= deadline {
10322            return;
10323        }
10324        tokio::time::sleep(poll).await;
10325    }
10326}
10327
10328/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10329/// admission that it could not even get a verify command to run, as opposed
10330/// to evidence the command actually produced? A caller that would otherwise
10331/// read a resource-blocked outcome as a red command must check this first:
10332/// see [`Runner::gate`], which retries rather than records `Blocked` when
10333/// this is true.
10334fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10335    outcomes.iter().any(|o| o.resource_blocked)
10336}
10337
10338/// What [`Runner::gate_fix_round`] decided.
10339pub(crate) enum GateFix {
10340    /// The tree changed and `verify.e2e` is still green: run the gate again.
10341    Retry,
10342    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10343    /// last failure stands and the run ends blocked.
10344    Stop,
10345    /// `verify.e2e` could not run after the fix (magi's own contention):
10346    /// decide nothing now, a later reentry retries.
10347    Defer,
10348}
10349
10350/// Is every red command in `outcomes` an ordinary failure the code could
10351/// explain: it ran, exited non-zero, and said something?
10352///
10353/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10354/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10355/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10356/// what the command is stays the gate's business.
10357fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10358    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10359    red.peek().is_some()
10360        && red.all(|o| {
10361            !o.resource_blocked
10362                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10363                && !o.output_tail.trim().is_empty()
10364        })
10365}
10366
10367/// Describe one verify command's outcome for the event log, distinguishing a
10368/// build/link failure — the toolchain never produced a binary to run — from
10369/// an actual test failure, since only the latter is a verdict on the patch.
10370fn e2e_outcome_label(o: &CommandOutcome) -> String {
10371    if o.ok() {
10372        return "pass".to_owned();
10373    }
10374    let reason = if o.build_failed() {
10375        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10376    } else {
10377        format!("FAIL ({:?})", o.code)
10378    };
10379    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10380}
10381
10382/// Run `verify.e2e`, retrying once if the first attempt could not build or
10383/// link — a build/link failure is frequently a race against a shared
10384/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10385/// `verify` event per command, tagged with `context` (normally `"round N"`)
10386/// so the two call sites that need this — the ordinary per-round leg in
10387/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10388/// it will ever call a round green — read identically in the event log.
10389async fn run_e2e_with_retry(
10390    state: &mut RunState,
10391    shell: &[String],
10392    commands: &[String],
10393    worktree: &Path,
10394    timeout: Duration,
10395    context: &str,
10396) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10397    let (mut e2e, mut timed_out_pids) = run_commands(
10398        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10399    )
10400    .await;
10401    for o in &e2e {
10402        state.event(
10403            "verify",
10404            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10405        );
10406    }
10407    // A build/link failure is not a verdict on the patch — it is frequently a
10408    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10409    // one retry before letting a red like that decide the round.
10410    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10411    if verify_retried {
10412        state.event(
10413            "verify",
10414            format!(
10415                "{context}: verify could not build/link, not a test result — retrying once \
10416                 before concluding"
10417            ),
10418        );
10419        let retried = run_commands(
10420            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10421        )
10422        .await;
10423        e2e = retried.0;
10424        // Both attempts' timeouts matter, not just the last one: the first
10425        // attempt's descendants may still be alive alongside the retry's.
10426        timed_out_pids.extend(retried.1);
10427        for o in &e2e {
10428            state.event(
10429                "verify",
10430                format!(
10431                    "{context}: retry `{}` -> {}",
10432                    o.command,
10433                    e2e_outcome_label(o)
10434                ),
10435            );
10436        }
10437    }
10438    (e2e, verify_retried, timed_out_pids)
10439}
10440
10441/// Run configured shell commands in `cwd`, in order. The second element is
10442/// the pid of every command that hit `timeout` and was still running when
10443/// this stopped waiting on it (best-effort: `None` when the platform did not
10444/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10445/// that releases a shared resource afterward needs to know.
10446///
10447/// Records `task` into [`RunState::active`] at every command boundary
10448/// (`RunState::task_command`) and clears it once the whole list has run
10449/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10450/// for minutes with no seat and no output of its own to show for it (see
10451/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10452/// apart from "not yet run" without this), and this is the only place that
10453/// knows which command is running right now and how many are left. Three
10454/// saves per command — start, not per second — matching the same "only at a
10455/// boundary" rule [`wave`] already follows for seats.
10456#[allow(clippy::too_many_arguments)]
10457async fn run_commands(
10458    state: &mut RunState,
10459    node: &str,
10460    task: &str,
10461    attempt: usize,
10462    shell: &[String],
10463    commands: &[String],
10464    cwd: &Path,
10465    timeout: Duration,
10466) -> (Vec<CommandOutcome>, Vec<u32>) {
10467    if commands.is_empty() {
10468        // Nothing to mark as running and nothing to clear — an empty list
10469        // means "not configured", and touching `active` (or the disk) over
10470        // that would be a write for every round of a repo with no
10471        // `verify.e2e` / `verify.gate` commands at all.
10472        return (Vec::new(), Vec::new());
10473    }
10474    let mut out = Vec::new();
10475    let mut timed_out_pids = Vec::new();
10476    let total = commands.len();
10477    for (idx, command) in commands.iter().enumerate() {
10478        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10479        if let Err(e) = state.save() {
10480            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10481        }
10482        let started = Instant::now();
10483        let mut cmd = tokio::process::Command::new(&shell[0]);
10484        cmd.quiet();
10485        cmd.args(&shell[1..])
10486            .arg(command)
10487            .current_dir(cwd)
10488            .stdin(std::process::Stdio::null())
10489            .stdout(std::process::Stdio::piped())
10490            .stderr(std::process::Stdio::piped())
10491            .kill_on_drop(true);
10492        let mut outputs = Vec::new();
10493        let protected = (|| -> Result<()> {
10494            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10495                outputs.push(guard);
10496            }
10497            if let Some(cache) = state.config.cache_dir() {
10498                outputs.push(crate::retention::OutputLease::cache(
10499                    &crate::run::home(),
10500                    &cache,
10501                    false,
10502                )?);
10503            }
10504            Ok(())
10505        })();
10506        if let Err(e) = protected {
10507            out.push(CommandOutcome {
10508                command: command.clone(),
10509                code: None,
10510                output_tail: format!("output ownership unavailable: {e:#}"),
10511                duration_ms: 0,
10512                resource_blocked: true,
10513            });
10514            continue;
10515        }
10516        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10517        let (code, body) = match spawned {
10518            Ok(mut running) => {
10519                let pid = running.child().id();
10520                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10521                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10522                let mut paths = vec![cwd.to_owned()];
10523                if let Some(cache) = state.config.cache_dir() {
10524                    paths.push(cache);
10525                }
10526                let mut pressure = None;
10527                let status = tokio::select! {
10528                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10529                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10530                        pressure = Some(reason); None
10531                    }
10532                };
10533                let (code, failure) = match status {
10534                    Some(Ok(Ok(status))) => (
10535                        status.code(),
10536                        status
10537                            .code()
10538                            .is_none()
10539                            .then(|| format!("command terminated: {status}")),
10540                    ),
10541                    Some(Ok(Err(error))) => {
10542                        (None, Some(format!("failed waiting for command: {error}")))
10543                    }
10544                    Some(Err(_)) => {
10545                        // Leased cleanup confirms process-tree death itself.
10546                        // Unleased commands still need the caller's PID check
10547                        // after kill-on-drop, before releasing cache ownership.
10548                        if !running.coordinated()
10549                            && let Some(pid) = pid
10550                        {
10551                            timed_out_pids.push(pid);
10552                        }
10553                        (
10554                            None,
10555                            Some(format!("timed out after {}s", timeout.as_secs())),
10556                        )
10557                    }
10558                    None => (None, None),
10559                };
10560                running.finish().await;
10561                if let Some(reason) = pressure {
10562                    state.resource_block = Some(crate::run::ResourceBlock {
10563                        node: node.to_owned(),
10564                        kind: "disk pressure".to_owned(),
10565                        detail: reason.clone(),
10566                        at: Timestamp::now(),
10567                    });
10568                    out.push(CommandOutcome {
10569                        command: command.clone(),
10570                        code: None,
10571                        output_tail: reason,
10572                        duration_ms: started.elapsed().as_millis() as u64,
10573                        resource_blocked: true,
10574                    });
10575                    break;
10576                }
10577                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10578                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10579                if let Some(failure) = failure {
10580                    body.push('\n');
10581                    body.push_str(&failure);
10582                }
10583                (code, body)
10584            }
10585            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10586        };
10587        out.push(CommandOutcome {
10588            command: command.clone(),
10589            code,
10590            output_tail: tail(&body, OUTPUT_TAIL),
10591            duration_ms: started.elapsed().as_millis() as u64,
10592            resource_blocked: false,
10593        });
10594    }
10595    state.task_finished(task);
10596    if let Err(e) = state.save() {
10597        tracing::warn!("could not persist the end of {task}: {e:#}");
10598    }
10599    (out, timed_out_pids)
10600}
10601
10602/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10603/// section (`report::run`) and in the `merge` event this node records — for
10604/// the operator to run by hand.
10605///
10606/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10607/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10608/// merge commits", or "require linear history") rejects the push a `--no-ff`
10609/// merge would produce, which is exactly the guidance this function replaces.
10610/// `message`'s first line becomes the squash commit's subject, matching the
10611/// note `report::run` prints alongside this command — see that function for
10612/// why an explicit subject is not optional there.
10613fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10614    let repo = repo.display();
10615    match style {
10616        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10617        MergeStyle::Squash => {
10618            // The subject sits inside double quotes, and a title an agent
10619            // wrote may carry the characters that break out of them.
10620            let subject = message
10621                .lines()
10622                .next()
10623                .unwrap_or(branch)
10624                .replace(['\\', '"', '$', '`'], "");
10625            format!(
10626                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10627            )
10628        }
10629        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10630    }
10631}
10632
10633/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10634/// under the hood, rejects a `title` over 256 characters and the whole
10635/// command fails — no PR at all, for a run whose body was otherwise fine
10636/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10637/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10638/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10639/// this bound. It is a margin, not a guarantee — a title packed
10640/// with multi-unit characters could still in principle land close to the
10641/// edge, but a real task title's occasional emoji or accented letter fits
10642/// comfortably inside it.
10643const PR_TITLE_MAX: usize = 240;
10644
10645/// A pull request title from the opening line of `text`, or `None` when that
10646/// line is not English (GitHub text is) or has no letters.
10647///
10648/// A line within `max` is kept as is. A longer one is cut at the end of its
10649/// first sentence when that falls inside `max`, else at a word boundary with a
10650/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10651/// make every merely-truncated title look non-English). The language check
10652/// runs on the kept text before any mark is added, so only what GitHub will
10653/// show is judged: an English opening followed by non-ASCII far past the cut
10654/// still passes.
10655fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10656    let line = queue::first_line(text)?;
10657    let chars: Vec<char> = line.chars().collect();
10658    let (kept, mark) = if chars.len() <= max {
10659        (line.to_owned(), "")
10660    } else if let Some(end) = sentence_end(&chars, max) {
10661        (chars[..end].iter().collect::<String>(), "")
10662    } else {
10663        let room = max.saturating_sub(3);
10664        // Cut at the last space inside the room; when the char just past the
10665        // room is a space the room already ends on a word.
10666        let cut = if chars[room].is_whitespace() {
10667            room
10668        } else {
10669            chars[..room]
10670                .iter()
10671                .rposition(|c| c.is_whitespace())
10672                .unwrap_or(room)
10673        };
10674        let head: String = chars[..cut].iter().collect();
10675        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10676        (head.to_owned(), "...")
10677    };
10678    if kept.is_empty() {
10679        return None;
10680    }
10681    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10682        return None;
10683    }
10684    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10685        return None;
10686    }
10687    Some(format!("{kept}{mark}"))
10688}
10689
10690/// The char length of the first sentence of `chars` when it ends within `max`
10691/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10692/// abbreviations so `e.g. foo` does not end a title early.
10693fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10694    const MIN: usize = 20;
10695    for i in MIN..max.min(chars.len()) {
10696        if !matches!(chars[i], '.' | '!' | '?') {
10697            continue;
10698        }
10699        let Some(&next) = chars.get(i + 1) else {
10700            continue;
10701        };
10702        if !next.is_whitespace() {
10703            continue;
10704        }
10705        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10706        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10707            continue;
10708        }
10709        let word: String = chars[..i]
10710            .iter()
10711            .rev()
10712            .take_while(|c| !c.is_whitespace())
10713            .collect::<Vec<_>>()
10714            .into_iter()
10715            .rev()
10716            .collect();
10717        let word = word.to_ascii_lowercase();
10718        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10719            continue;
10720        }
10721        let end = chars[..i]
10722            .iter()
10723            .rposition(|c| !c.is_whitespace())
10724            .map_or(i, |p| p + 1);
10725        return Some(end);
10726    }
10727    None
10728}
10729
10730/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10731/// change: a title and a body describing what was *implemented*, not the task
10732/// that asked for it. A task reads as a request; a reader of the merged
10733/// history wants the change.
10734struct PrMessage {
10735    title: String,
10736    body: String,
10737}
10738
10739impl PrMessage {
10740    /// Title, blank line, body. The first line is the squash/merge commit
10741    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10742    /// has to stay one sensible line.
10743    fn commit_message(&self) -> String {
10744        format!("{}\n\n{}", self.title, self.body)
10745    }
10746}
10747
10748/// The text after a leading `TITLE:` (any case) on `line`.
10749fn title_marker(line: &str) -> Option<&str> {
10750    let line = line.trim();
10751    let head = line.get(..6)?;
10752    head.eq_ignore_ascii_case("title:")
10753        .then(|| line[6..].trim())
10754}
10755
10756/// The implementer's own one-line title: the `TITLE:` line the implement
10757/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10758/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10759/// source, and a title that says as much is refused here too.
10760fn summary_title(summary: &str) -> Option<String> {
10761    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10762    let raw = title_marker(first)?;
10763    if raw.is_empty() {
10764        return None;
10765    }
10766    let title = queue::title_from(raw, PR_TITLE_MAX);
10767    let lower = title.to_ascii_lowercase();
10768    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10769        return None;
10770    }
10771    Some(title)
10772}
10773
10774/// How `open_review`'s instruction begins; see [`landing_title`].
10775const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10776
10777/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10778pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10779
10780/// Does this run review work that already existed, rather than implement a
10781/// task? Runs recorded before `reviewed_commits` existed carry only the
10782/// candidate marker.
10783pub(crate) fn is_review_run(state: &RunState) -> bool {
10784    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10785}
10786
10787/// The title of a review-only run: the subject of the oldest commit under
10788/// review. Later commits are usually fixups, and `instruction` is the review
10789/// prompt, which says nothing about the change. GitHub text is English, so a
10790/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10791fn review_title(state: &RunState) -> Option<String> {
10792    english_subject(
10793        state.reviewed_commits.as_ref()?.first()?,
10794        state.config.graph.github_english(),
10795    )
10796}
10797
10798/// `raw` as a pull request title, or `None` when it is blank, not English
10799/// (GitHub text is), or one of magi's own candidate commit subjects.
10800fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10801    let raw = raw.trim();
10802    if raw.is_empty() {
10803        return None;
10804    }
10805    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10806        return None;
10807    }
10808    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10809        return None;
10810    }
10811    let title = queue::title_from(raw, PR_TITLE_MAX);
10812    let lower = title.to_ascii_lowercase();
10813    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10814        return None;
10815    }
10816    Some(title)
10817}
10818
10819/// What a review-only run's branch says about itself, read at the moment the
10820/// pull request is opened.
10821#[derive(Debug, Clone, PartialEq, Eq)]
10822struct BranchFacts {
10823    /// `(subject, body)` of each commit, oldest first.
10824    commits: Vec<(String, String)>,
10825    /// Trimmed `git diff --stat`.
10826    stat: String,
10827}
10828
10829/// Longest diff stat shown: this many file lines plus the summary line.
10830const STAT_FILE_LINES: usize = 25;
10831/// Cap on the commit list, well inside GitHub's 65536-character body limit.
10832const COMMITS_MAX_CHARS: usize = 20_000;
10833
10834/// Read the commits and diff stat of `base..branch`. `None` when git cannot
10835/// say or finds nothing, so the caller falls back to what the run recorded.
10836async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10837    let commits = git::commit_log(repo, base, branch).await.ok()?;
10838    if commits.is_empty() {
10839        return None;
10840    }
10841    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10842    let lines: Vec<&str> = stat.lines().collect();
10843    let stat = if lines.len() > STAT_FILE_LINES + 1 {
10844        let omitted = lines.len() - 1 - STAT_FILE_LINES;
10845        let more = format!(" ... {omitted} more file(s)");
10846        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10847        kept.push(&more);
10848        kept.push(lines[lines.len() - 1]);
10849        kept.join("\n")
10850    } else {
10851        lines.join("\n")
10852    };
10853    Some(BranchFacts { commits, stat })
10854}
10855
10856/// Defang what would break the surrounding markdown: a closing `</details>`
10857/// and a code fence.
10858fn markdown_safe(text: &str) -> String {
10859    text.replace("</details>", "&lt;/details&gt;")
10860        .replace("\x60\x60\x60", "~~~")
10861}
10862
10863fn neutral_title(state: &RunState, winner: char) -> String {
10864    format!(
10865        "chore: land candidate {} of run {}",
10866        winner.to_ascii_uppercase(),
10867        state.id
10868    )
10869}
10870
10871/// Reuse the graph's bounded fixer without starting another graph or driver.
10872/// Copy the working state back even on errors so spent rounds remain visible.
10873pub(crate) async fn repair_landing_gate(
10874    state: &mut RunState,
10875    winner: &Candidate,
10876    outcomes: &[CommandOutcome],
10877) -> Result<GateFix> {
10878    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10879        return Ok(GateFix::Stop);
10880    }
10881    let mut runner = Runner {
10882        roles: state.config.resolve_roles()?,
10883        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10884        state: state.clone(),
10885        pause: Pause::new(),
10886        interrupt: Pause::new(),
10887    };
10888    let result = runner.gate_fix_round(winner, outcomes).await;
10889    *state = runner.state;
10890    result
10891}
10892
10893/// Recheck a repair's e2e requirement across failures, resource deferrals and
10894/// process interruptions. A successful result only applies to its exact SHA.
10895pub(crate) async fn check_gate_repair_e2e(
10896    state: &mut RunState,
10897    winner: &Candidate,
10898    wait: &crate::repo_lock::WaitPolicy,
10899) -> Result<GateFix> {
10900    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10901        return Ok(GateFix::Retry);
10902    }
10903    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10904    if state
10905        .gate_repair_e2e
10906        .as_ref()
10907        .is_some_and(|r| r.head == head && r.passed)
10908    {
10909        return Ok(GateFix::Retry);
10910    }
10911    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10912        head: head.clone(),
10913        passed: false,
10914        outcomes: Vec::new(),
10915    });
10916    state.save()?;
10917    let commands = state.config.verify.e2e.clone();
10918    let outcomes = if commands.is_empty() {
10919        Vec::new()
10920    } else {
10921        let shell = state.config.shell();
10922        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10923        let cache_dir = state.config.cache_dir();
10924        let (outcomes, _) = with_cache_lease(
10925            state,
10926            cache_dir.as_deref(),
10927            "e2e",
10928            "e2e",
10929            &winner.worktree,
10930            &head,
10931            timeout,
10932            "gate repair e2e",
10933            wait,
10934            |state, budget| async move {
10935                run_e2e_with_retry(
10936                    state,
10937                    &shell,
10938                    &commands,
10939                    &winner.worktree,
10940                    budget,
10941                    "gate repair e2e",
10942                )
10943                .await
10944            },
10945        )
10946        .await?;
10947        outcomes
10948    };
10949    let deferred = verify_inconclusive(&outcomes);
10950    let passed = outcomes.iter().all(CommandOutcome::ok)
10951        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10952    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10953        head,
10954        passed,
10955        outcomes,
10956    });
10957    state.save()?;
10958    if deferred {
10959        Ok(GateFix::Defer)
10960    } else if !passed {
10961        state.event(
10962            "gate",
10963            "verify.e2e failed after gate repair; nothing published",
10964        );
10965        Ok(GateFix::Stop)
10966    } else {
10967        Ok(GateFix::Retry)
10968    }
10969}
10970
10971/// Run the same configured gate for a landing head and a fresh graph.
10972pub(crate) async fn run_landing_gate(
10973    state: &mut RunState,
10974    winner: &Candidate,
10975    wait: &crate::repo_lock::WaitPolicy,
10976) -> Result<Vec<CommandOutcome>> {
10977    let shell = state.config.shell();
10978    let gate_commands = state.config.verify.gate.clone();
10979    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10980    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10981        Vec::new()
10982    } else {
10983        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10984        let cache_dir = state.config.cache_dir();
10985        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10986        let (outcomes, _) = with_cache_lease(
10987            state,
10988            cache_dir.as_deref(),
10989            "gate",
10990            "gate",
10991            &winner.worktree,
10992            &head,
10993            timeout,
10994            "final gate",
10995            wait,
10996            |state, budget| {
10997                let shell = shell.clone();
10998                let gate_commands = gate_commands.clone();
10999                let winner = winner.clone();
11000                async move {
11001                    Runner::run_pre_gate(state, &winner).await;
11002                    let (outcomes, timed_out_pids) = run_commands(
11003                        state,
11004                        "gate",
11005                        "gate",
11006                        0,
11007                        &shell,
11008                        &gate_commands,
11009                        &winner.worktree,
11010                        budget,
11011                    )
11012                    .await;
11013                    (outcomes, false, timed_out_pids)
11014                }
11015            },
11016        )
11017        .await?;
11018        outcomes
11019    };
11020    if outcomes.is_empty() {
11021        // Nothing configured to check — distinct from every other
11022        // silence in this run's event log, since an empty `gate` alone
11023        // no longer says whether the gate ran at all (see
11024        // `RunState::gate_ran`'s own doc).
11025        state.event("gate", "no gate commands configured; not tested");
11026    }
11027    for o in &outcomes {
11028        state.event(
11029            "gate",
11030            format!(
11031                "`{}` -> {}",
11032                o.command,
11033                if o.ok() {
11034                    "pass".to_owned()
11035                } else {
11036                    format!(
11037                        "FAIL ({:?})\n{}",
11038                        o.code,
11039                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11040                    )
11041                }
11042            ),
11043        );
11044    }
11045    Ok(outcomes)
11046}
11047
11048/// The pull request title to hand to `land::merge_subject`. A review-only run
11049/// opened by an earlier build titled its pull request with the review prompt;
11050/// that title is dropped (empty, so the fallback applies) rather than landed.
11051/// Any other title, including an operator's rename, passes through untouched,
11052/// and so does every title of a run that implements a task.
11053pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11054    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11055        ""
11056    } else {
11057        pr_title
11058    }
11059}
11060
11061/// What the squash subject falls back to when the pull request title is empty
11062/// or candidate-shaped: for a review-only run the derived title, never the
11063/// review prompt held in `instruction`.
11064pub fn landing_subject_source(state: &RunState) -> String {
11065    if is_review_run(state) {
11066        let winner = state.candidates.first().map_or('A', |c| c.label);
11067        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11068    }
11069    state.instruction.clone()
11070}
11071
11072/// `summary` without its `TITLE:` line, which the pull request title already
11073/// carries.
11074fn summary_without_title(summary: &str) -> String {
11075    let mut lines = summary.trim().lines().peekable();
11076    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11077        lines.next();
11078    }
11079    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11080}
11081
11082/// The pull request title and body for the winning candidate.
11083///
11084/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11085/// the task's own opening line via [`queue::title_from`] when there is none.
11086/// `state.instruction` can open with blank lines (`task_text` only rejects a
11087/// body that is blank *entirely*), which `title_from` skips.
11088///
11089/// Publishing uses an independent description generated from the final diff
11090/// and structured evidence. Internal summaries and reports remain local.
11091/// Non-publishing previews may display the original task and candidate summary.
11092/// The footer retains the run and candidate provenance.
11093#[cfg(test)]
11094fn pr_message(state: &RunState, winner: char) -> PrMessage {
11095    pr_message_with(state, winner, None)
11096}
11097
11098/// [`pr_message`] with what the branch of a review-only run says about itself.
11099/// `facts` is ignored for a run that implements a task.
11100#[cfg(test)]
11101fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11102    let raw = pr_message_raw(
11103        state,
11104        winner,
11105        facts,
11106        crate::pr_outcome::FinalHead::Unreported,
11107        ChangeNote::NotAsked,
11108    );
11109    let id = crate::scrub::Identity::current();
11110    PrMessage {
11111        title: crate::scrub::scrub(&raw.title, &id),
11112        body: crate::scrub::scrub(&raw.body, &id),
11113    }
11114}
11115
11116/// What the public description may say about the change itself.
11117#[derive(Debug, Clone, Copy)]
11118enum ChangeNote<'a> {
11119    /// A local preview or non-publishing merge mode: no public generation needed.
11120    NotAsked,
11121    /// A summary written from the final diff.
11122    Summary(&'a crate::run::ChangeSummary),
11123    /// A summary was asked for and could not be produced; the text says why.
11124    Unavailable(&'a str),
11125}
11126
11127fn pr_message_raw(
11128    state: &RunState,
11129    winner: char,
11130    facts: Option<&BranchFacts>,
11131    head: crate::pr_outcome::FinalHead<'_>,
11132    change: ChangeNote<'_>,
11133) -> PrMessage {
11134    if !matches!(change, ChangeNote::NotAsked) {
11135        let (title, mut body) = match change {
11136            ChangeNote::Summary(c) => (
11137                c.title.clone(),
11138                format!("## Summary\n\n{}\n\n", c.body.trim()),
11139            ),
11140            ChangeNote::Unavailable(why) => (
11141                neutral_title(state, winner),
11142                format!(
11143                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11144                ),
11145            ),
11146            ChangeNote::NotAsked => unreachable!(),
11147        };
11148        body.push_str(&public_pr_evidence(state, head));
11149        body.push_str(&format!(
11150            "\n---\nmagi:run/{} magi:candidate-{}\n",
11151            state.id,
11152            winner.to_ascii_lowercase()
11153        ));
11154        return PrMessage { title, body };
11155    }
11156    pr_preview(state, winner, facts, head)
11157}
11158
11159/// Append public facts from the run records after every prose generation,
11160/// including posting-gate rewrites. The agent cannot drop this provenance.
11161fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11162    let mut body = crate::pr_outcome::review_rounds(state);
11163    body.push_str(&crate::pr_outcome::final_verification(state, head));
11164    let open = state.last_round_findings();
11165    if !open.is_empty() {
11166        body.push_str("## Open review findings\n\n");
11167        for f in open {
11168            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11169        }
11170        body.push('\n');
11171    }
11172    if let Some(c) = state.change_summary.as_ref()
11173        && head == crate::pr_outcome::FinalHead::At(&c.head)
11174    {
11175        body.push_str(&format!(
11176            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11177            c.base, c.head
11178        ));
11179    }
11180    body
11181}
11182
11183/// Local-only preview of the operator's task and candidate summary.
11184fn pr_preview(
11185    state: &RunState,
11186    winner: char,
11187    facts: Option<&BranchFacts>,
11188    head: crate::pr_outcome::FinalHead<'_>,
11189) -> PrMessage {
11190    let summary = state
11191        .candidates
11192        .iter()
11193        .find(|c| c.label == winner)
11194        .map(|c| c.summary.as_str())
11195        .unwrap_or_default();
11196    // The fallback is the operator's own words and may not be English; GitHub
11197    // text always is, so a non-English task gets a neutral title instead.
11198    let review = is_review_run(state);
11199    let title = if review {
11200        facts
11201            .and_then(|f| {
11202                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11203            })
11204            .or_else(|| review_title(state))
11205            .or_else(|| {
11206                state
11207                    .candidates
11208                    .iter()
11209                    .find(|c| c.label == winner)
11210                    .filter(|c| !c.branch.starts_with("magi/"))
11211                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11212            })
11213            .unwrap_or_else(|| neutral_title(state, winner))
11214    } else {
11215        summary_title(summary).unwrap_or_else(|| {
11216            english_title(
11217                &state.instruction,
11218                PR_TITLE_MAX,
11219                state.config.graph.github_english(),
11220            )
11221            .unwrap_or_else(|| neutral_title(state, winner))
11222        })
11223    };
11224
11225    let mut body = String::new();
11226    let what = summary_without_title(summary);
11227    if !what.is_empty() {
11228        body.push_str("## Summary\n\n");
11229        body.push_str(&what);
11230        body.push_str("\n\n");
11231    }
11232
11233    // Review history uses structured facts across all rounds, never reports.
11234    body.push_str(&crate::pr_outcome::review_rounds(state));
11235    body.push_str(&crate::pr_outcome::final_verification(state, head));
11236    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11237
11238    let open = state.open_findings();
11239    if !open.is_empty() {
11240        body.push_str("## Open review findings\n\n");
11241        for f in &open {
11242            body.push_str(&format!(
11243                "- `{}` [{:?}] {}\n",
11244                f.id, f.severity, "Context retained in local review records."
11245            ));
11246        }
11247        body.push('\n');
11248    }
11249
11250    if let Some(fix) = fix
11251        && !fix.rejected.is_empty()
11252    {
11253        body.push_str("## Declined by the fixer\n\n");
11254        for r in &fix.rejected {
11255            body.push_str(&format!(
11256                "- `{}`: {}\n",
11257                r.id, "Reason retained in local review records."
11258            ));
11259        }
11260        body.push('\n');
11261    }
11262
11263    if review {
11264        // The review prompt is not the task; list what the branch carries.
11265        body.push_str("## Commits under review\n\n");
11266        if let Some(facts) = facts {
11267            let mut left = COMMITS_MAX_CHARS;
11268            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11269                let mut entry = format!("- {}\n", markdown_safe(subject));
11270                for l in markdown_safe(text).lines() {
11271                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11272                }
11273                if left == 0 {
11274                    body.push_str(&format!(
11275                        "- ... {} more commit(s)\n",
11276                        facts.commits.len() - i
11277                    ));
11278                    break;
11279                }
11280                if entry.len() > left {
11281                    // Even the first commit is cut: one huge body must not
11282                    // push the whole description past GitHub's limit.
11283                    let mut end = left;
11284                    while !entry.is_char_boundary(end) {
11285                        end -= 1;
11286                    }
11287                    entry.truncate(end);
11288                    entry.push_str("\n  ... (truncated)\n");
11289                    left = 0;
11290                } else {
11291                    left -= entry.len();
11292                }
11293                body.push_str(&entry);
11294            }
11295            if !facts.stat.trim().is_empty() {
11296                body.push_str(&format!(
11297                    "\n## Diff stat\n\n```\n{}\n```\n",
11298                    markdown_safe(facts.stat.trim())
11299                ));
11300            }
11301        } else {
11302            match &state.reviewed_commits {
11303                Some(subjects) => {
11304                    for s in subjects {
11305                        body.push_str(&format!("- {}\n", s.trim()));
11306                    }
11307                }
11308                None => {
11309                    // An older run kept only the prompt, with the commit list
11310                    // after its first paragraph.
11311                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11312                    body.push_str(rest.trim());
11313                    body.push('\n');
11314                }
11315            }
11316        }
11317    } else {
11318        let task = state.instruction.trim();
11319        let task = if task.is_empty() {
11320            "(empty task)"
11321        } else {
11322            task
11323        };
11324        body.push_str(&format!(
11325            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11326            task.replace("</details>", "&lt;/details&gt;")
11327        ));
11328    }
11329
11330    body.push_str(&format!(
11331        "\n---\nmagi:run/{} magi:candidate-{}\n",
11332        state.id,
11333        winner.to_ascii_lowercase()
11334    ));
11335
11336    PrMessage { title, body }
11337}
11338
11339/// Source imports have already been prepared from structured inputs. Legacy
11340/// inferred references must never instruct a seat to reconstruct those imports.
11341fn seeded_instruction(state: &RunState) -> String {
11342    let mut inputs = Vec::new();
11343    if let Some(start) = &state.sources.start_from {
11344        inputs.push(format!(
11345            "Start from `{}` at `{}` (including ancestry).",
11346            start.token, start.sha
11347        ));
11348    }
11349    for pick in &state.sources.pick_commits {
11350        inputs.push(format!(
11351            "Pick `{}` at `{}` (single commit).",
11352            pick.token, pick.sha
11353        ));
11354    }
11355    if inputs.is_empty() {
11356        state.instruction.clone()
11357    } else {
11358        format!(
11359            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11360            state.instruction,
11361            inputs.join("\n")
11362        )
11363    }
11364}
11365
11366/// Does the winner have no commits ahead of the base it would land on?
11367/// Any failure to find out reads as "not empty": the merge then behaves as it
11368/// always did rather than refusing on a guess.
11369async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11370    let base = &state.base_branch;
11371    let mut against = base.clone();
11372    if mode == MergeMode::Pr {
11373        // A pull request lands on the remote's base, never the local branch
11374        // of the same name: if that cannot be read, "not empty" is the safe
11375        // answer.
11376        let remote = &state.config.merge.remote;
11377        let tracking = format!("{remote}/{base}");
11378        let fetched = git::fetch(repo, remote, base).await;
11379        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11380            against = tracking;
11381        } else {
11382            return false;
11383        }
11384    }
11385    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11386}
11387
11388/// Why nothing was opened for an empty winner, with what the task's own
11389/// references resolved to.
11390fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11391    let mut detail = format!(
11392        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11393         nothing to open a pull request for"
11394    );
11395    detail.push_str(
11396        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11397    );
11398    detail
11399}
11400
11401/// What the `Pr` merge does once it knows whether the branch already has an
11402/// open pull request.
11403#[derive(Debug, PartialEq, Eq)]
11404enum PrPlan {
11405    Create,
11406    Adopt { url: String, title: String },
11407    Stop(String),
11408}
11409
11410/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11411/// a failed lookup -> stop with the real reason. Never guesses.
11412fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11413    match found {
11414        Ok(land::OpenPr::None) => PrPlan::Create,
11415        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11416        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11417            "several open pull requests exist for this branch, not picking one: {}",
11418            urls.join(" ")
11419        )),
11420        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11421    }
11422}
11423
11424/// `gh pr create`, returning the PR url.
11425pub(crate) async fn gh_pr_create(
11426    cwd: &Path,
11427    base: &str,
11428    head: &str,
11429    title: &str,
11430    body: &str,
11431) -> Result<String> {
11432    let out = tokio::process::Command::new("gh")
11433        .args([
11434            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11435        ])
11436        .current_dir(cwd)
11437        .quiet()
11438        .stdin(std::process::Stdio::null())
11439        .output()
11440        .await
11441        .context("spawn gh")?;
11442    if out.status.success() {
11443        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11444    } else {
11445        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11446    }
11447}
11448
11449/// Never force deletion of uncommitted work, including unknown legacy trees.
11450async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11451    match std::fs::symlink_metadata(path) {
11452        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11453            // A previous cleanup may have removed the tree before a branch
11454            // deletion or state save failed. Unregister it if still present.
11455            git::worktree_remove_clean(repo, path).await?;
11456            return Ok(());
11457        }
11458        Err(e) => return Err(e.into()),
11459        Ok(_) => {}
11460    }
11461    if !git::is_clean(path).await? {
11462        bail!(
11463            "worktree has uncommitted work; refusing removal: {}",
11464            path.display()
11465        );
11466    }
11467    if !git::worktree_remove_clean(repo, path).await? {
11468        bail!("worktree removal refused: {}", path.display());
11469    }
11470    Ok(())
11471}
11472
11473/// Tear a run's worktrees and branches down.
11474///
11475/// `home` is where the updated `run.json` is saved (via
11476/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11477/// a housekeeping pass already has its own honest `home` handed to it, and
11478/// falling through to the global here would write back through whichever
11479/// directory some other process or test pinned into that `OnceLock` first,
11480/// not the one the caller actually resolved its `runs` and `state` from.
11481pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11482    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11483        bail!("refusing fold: {reason}");
11484    }
11485    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11486        bail!("candidate preservation incomplete; refusing cleanup");
11487    }
11488    // Serialize ordinary folding with the same starts/resumes as target-only
11489    // eviction. Otherwise a second janitor could remove the whole worktree
11490    // around a live output borrower even though target eviction was fenced.
11491    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11492    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11493    let _driver = crate::driver::maintenance(home, &state.id)?;
11494    let mut output_guards = Vec::new();
11495    for entry in std::fs::read_dir(state.worktree_root())
11496        .into_iter()
11497        .flatten()
11498        .flatten()
11499    {
11500        if entry.path().join(".git").is_file() {
11501            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11502        }
11503    }
11504    let repo = state.repo.clone();
11505    // CLI, Web and automatic cleanup all enter here, independently of
11506    // Runner::fold_losers. Protect deletion and shared config release too.
11507    let wait = repository_wait(state, None);
11508    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11509        Ok(lease) => lease,
11510        Err(e) => {
11511            state.save_under(home)?;
11512            return Err(e);
11513        }
11514    };
11515    // Confirmed merges dispose of their owned sources after durable evidence
11516    // capture. Preflight every source before removing anything.
11517    let _source_outputs = if state.status == RunStatus::Merged
11518        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11519    {
11520        verify_merged_fold_sources(state, drop_winner, home).await?
11521    } else {
11522        Vec::new()
11523    };
11524    let root = state.worktree_root();
11525    let winner = state.tally.as_ref().map(|t| t.winner);
11526    let mut removed = Vec::new();
11527
11528    for i in 0..state.candidates.len() {
11529        let c = state.candidates[i].clone();
11530        let is_winner = Some(c.label) == winner;
11531        if is_winner && !drop_winner {
11532            continue;
11533        }
11534        let existed = c.worktree.exists();
11535        // A vanished directory can still have a Git registration that keeps
11536        // its branch checked out. Remove that registration before the branch.
11537        remove_preserved_worktree(&repo, &c.worktree).await?;
11538        if existed {
11539            removed.push(c.worktree.to_string_lossy().into_owned());
11540        }
11541        // A branch handed to a later run (and its pull request) is not this
11542        // run's to delete.
11543        let handed_over = state.released_branches.contains(&c.branch);
11544        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11545            if !git::branch_delete(&repo, &c.branch).await? {
11546                bail!("candidate branch removal refused");
11547            }
11548            removed.push(c.branch.clone());
11549        }
11550        state.candidates[i].folded = true;
11551    }
11552
11553    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11554        let path = name.path();
11555        let keep = !drop_winner
11556            && winner.is_some_and(|w| {
11557                path.file_name()
11558                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11559            });
11560        if keep {
11561            continue;
11562        }
11563        remove_preserved_worktree(&repo, &path).await?;
11564        removed.push(path.to_string_lossy().into_owned());
11565    }
11566
11567    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11568    // judge worktrees, so once the loop above has cleared all of them out,
11569    // the parent is a bare directory nobody else was ever going to remove -
11570    // git only ever managed what was inside it. Left alone, one of these
11571    // accumulates per fully-folded run; the operator's own machine had 74.
11572    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11573    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11574    // so does anything a run never claimed that happens to share the bay.
11575    remove_if_empty(&root);
11576
11577    if state.enabled_worktree_config && drop_winner {
11578        // A release, not a raw disable: some sibling run in this repository
11579        // may still hold its own reference (see `git::acquire_worktree_config`),
11580        // and only the last release actually turns the setting back off.
11581        git::release_worktree_config(&repo).await.ok();
11582        state.enabled_worktree_config = false;
11583    }
11584    state.save_under(home)?;
11585    Ok(removed)
11586}
11587
11588async fn verify_merged_fold_sources(
11589    state: &RunState,
11590    drop_winner: bool,
11591    home: &Path,
11592) -> Result<Vec<std::fs::File>> {
11593    let winner = state.tally.as_ref().map(|t| t.winner);
11594    let mut heads = BTreeSet::new();
11595    for candidate in &state.candidates {
11596        if Some(candidate.label) == winner && !drop_winner {
11597            continue;
11598        }
11599        anyhow::ensure!(
11600            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11601            "candidate recovery source is outside the inventoried run root: {}",
11602            candidate.worktree.display()
11603        );
11604        match std::fs::symlink_metadata(&candidate.worktree) {
11605            Ok(_) => {}
11606            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11607            Err(error) => return Err(error.into()),
11608        }
11609        if !state.released_branches.contains(&candidate.branch)
11610            && git::branch_exists(&state.repo, &candidate.branch).await?
11611        {
11612            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11613        }
11614    }
11615    let mut cargo_guards = Vec::new();
11616    let entries = match std::fs::read_dir(state.worktree_root()) {
11617        Ok(entries) => Some(entries),
11618        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11619        Err(error) => return Err(error.into()),
11620    };
11621    for entry in entries.into_iter().flatten() {
11622        let path = entry?.path();
11623        let metadata = std::fs::symlink_metadata(&path)?;
11624        anyhow::ensure!(
11625            metadata.is_dir() && !metadata.file_type().is_symlink(),
11626            "non-directory or symlink recovery source retained: {}",
11627            path.display()
11628        );
11629        if !drop_winner
11630            && winner.is_some_and(|w| {
11631                path.file_name()
11632                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11633            })
11634        {
11635            continue;
11636        }
11637        anyhow::ensure!(
11638            path.join(".git").is_file(),
11639            "unregistered recovery source retained: {}",
11640            path.display()
11641        );
11642        anyhow::ensure!(
11643            git::git(
11644                &path,
11645                &[
11646                    "status",
11647                    "--porcelain",
11648                    "--untracked-files=all",
11649                    "--ignored",
11650                    "--",
11651                    ".",
11652                    ":(exclude)target"
11653                ]
11654            )
11655            .await?
11656            .is_empty(),
11657            "source changes or ignored recovery files retained: {}",
11658            path.display()
11659        );
11660        match std::fs::symlink_metadata(path.join("target")) {
11661            Ok(_) => {
11662                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11663            }
11664            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11665            Err(error) => return Err(error.into()),
11666        }
11667        for checkpoint in [
11668            "rebase-merge",
11669            "rebase-apply",
11670            "sequencer",
11671            "CHERRY_PICK_HEAD",
11672            "MERGE_HEAD",
11673            "REBASE_HEAD",
11674            "REVERT_HEAD",
11675            "BISECT_LOG",
11676            "AUTO_MERGE",
11677            "MERGE_MSG",
11678        ] {
11679            let location = git::git(
11680                &path,
11681                &[
11682                    "rev-parse",
11683                    "--path-format=absolute",
11684                    "--git-path",
11685                    checkpoint,
11686                ],
11687            )
11688            .await?;
11689            anyhow::ensure!(
11690                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11691                if e.kind() == std::io::ErrorKind::NotFound),
11692                "Git recovery checkpoint retained: {checkpoint}"
11693            );
11694        }
11695        heads.insert(git::rev_parse(&path, "HEAD").await?);
11696    }
11697    crate::fold_evidence::preserve(state, home, &heads).await?;
11698
11699    Ok(cargo_guards)
11700}
11701
11702/// Remove `dir` if it exists and has nothing in it.
11703///
11704/// Best-effort and silent by design: a directory that is not empty (a run
11705/// whose winner is still parked there, a stray file some other process left)
11706/// is exactly the case this must refuse, and a directory that is already gone
11707/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11708/// already refuses a non-empty directory, so the emptiness check below is
11709/// belt, not suspenders - it is what keeps this from ever attempting the
11710/// removal in the case that matters, rather than trusting `remove_dir`'s
11711/// error path to have no side effects if it ever changed.
11712fn remove_if_empty(dir: &Path) {
11713    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11714        std::fs::remove_dir(dir).ok();
11715    }
11716}
11717
11718/// Severity of the worst open finding in the last review round, for reporting.
11719pub fn worst_open(state: &RunState) -> Option<Severity> {
11720    state
11721        .reviews
11722        .last()?
11723        .reviews
11724        .iter()
11725        .flat_map(|r| r.findings.iter())
11726        .map(|f| f.severity)
11727        .max()
11728}
11729
11730/// `[worktree] setup` hides its products per worktree, which needs
11731/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11732/// worktrees, like the hook's; every place that runs setup calls this first.
11733async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11734    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11735        git::acquire_worktree_config(repo).await?;
11736        state.enabled_worktree_config = true;
11737    }
11738    Ok(())
11739}
11740
11741#[cfg(test)]
11742mod tests {
11743    #[tokio::test]
11744    async fn launcher_base_fetch_requires_remote_success() {
11745        let tmp = tempfile::tempdir().unwrap();
11746        let repo = tmp.path().join("repo");
11747        let remote = tmp.path().join("remote");
11748        std::fs::create_dir(&repo).unwrap();
11749        std::fs::create_dir(&remote).unwrap();
11750        git::git(&remote, &["init", "--bare", "-b", "main"])
11751            .await
11752            .unwrap();
11753        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11754        git::git(
11755            &repo,
11756            &[
11757                "-c",
11758                "user.name=Fixture",
11759                "-c",
11760                "user.email=fixture@example.invalid",
11761                "commit",
11762                "--allow-empty",
11763                "-m",
11764                "fixture",
11765            ],
11766        )
11767        .await
11768        .unwrap();
11769        git::git(
11770            &repo,
11771            &["remote", "add", "origin", remote.to_str().unwrap()],
11772        )
11773        .await
11774        .unwrap();
11775        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11776        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11777        assert_eq!(
11778            resolve_base(&repo, "main", "origin", None)
11779                .await
11780                .unwrap()
11781                .commit,
11782            head
11783        );
11784        #[cfg(unix)]
11785        {
11786            let policy: crate::config::Coordination = toml::from_str(
11787                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11788            ).unwrap();
11789            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11790            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11791            let held = crate::repo_lock::SeatLease::operation_with(
11792                &repo,
11793                Some(&policy),
11794                &crate::repo_lock::WaitPolicy::default(),
11795            )
11796            .await
11797            .unwrap();
11798            assert!(held.is_some());
11799            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11800            tokio::pin!(fetch);
11801            assert!(
11802                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11803                    .await
11804                    .is_err()
11805            );
11806            assert!(
11807                !repo.join(".git/FETCH_HEAD").exists(),
11808                "no fetch before admission"
11809            );
11810            drop(held);
11811            let fetched = fetch.await.unwrap();
11812            assert_eq!(fetched.commit, head);
11813            assert_eq!(fetched.remote, "origin");
11814            assert_eq!(fetched.branch, "main");
11815        }
11816        // A surviving tracking ref is not successful fetch evidence.
11817        git::git(
11818            &repo,
11819            &[
11820                "remote",
11821                "set-url",
11822                "origin",
11823                tmp.path().join("missing").to_str().unwrap(),
11824            ],
11825        )
11826        .await
11827        .unwrap();
11828        let error = resolve_base(&repo, "main", "origin", None)
11829            .await
11830            .unwrap_err()
11831            .to_string();
11832        assert!(error.contains("cannot read origin/main"));
11833        assert!(error.contains("refusing to branch off"));
11834    }
11835
11836    #[test]
11837    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11838        let leaked = vec!["chore(deps): update a crate".to_owned()];
11839        let own = "fix(daemon): apply a chosen action";
11840        assert!(should_retitle("", own, &leaked));
11841        assert!(should_retitle(
11842            &format!("{REVIEW_PROMPT_OPENING} `x`"),
11843            own,
11844            &leaked
11845        ));
11846        assert!(should_retitle(
11847            "chore: land candidate A of run 1",
11848            own,
11849            &leaked
11850        ));
11851        assert!(should_retitle(
11852            "magi: candidate A (uncommitted work)",
11853            own,
11854            &leaked
11855        ));
11856        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11857        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11858        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11859    }
11860
11861    #[test]
11862    fn pr_merge_plan_creates_adopts_or_stops() {
11863        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11864        assert_eq!(
11865            pr_merge_plan(Ok(land::OpenPr::One {
11866                url: "u".into(),
11867                title: "t".into()
11868            })),
11869            PrPlan::Adopt {
11870                url: "u".into(),
11871                title: "t".into()
11872            }
11873        );
11874        let PrPlan::Stop(many) =
11875            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11876        else {
11877            panic!("many must stop");
11878        };
11879        assert!(many.contains('a') && many.contains('b'));
11880        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11881            panic!("a failed lookup must stop");
11882        };
11883        assert!(err.contains("bad token"));
11884    }
11885
11886    use super::*;
11887    use crate::run::GateStatus;
11888    use std::collections::BTreeMap;
11889    use std::time::Duration;
11890
11891    fn conductor() -> AgentSpec {
11892        AgentSpec {
11893            id: "conductor".to_owned(),
11894            kind: crate::config::AgentKind::Command,
11895            model: None,
11896            command: vec!["true".to_owned()],
11897            extra_args: Vec::new(),
11898            env: BTreeMap::new(),
11899            prompt_delivery: None,
11900            max_turn_secs: None,
11901        }
11902    }
11903
11904    fn spec(id: &str) -> AgentSpec {
11905        AgentSpec {
11906            id: id.to_owned(),
11907            kind: crate::config::AgentKind::Command,
11908            model: None,
11909            command: vec!["true".to_owned()],
11910            extra_args: Vec::new(),
11911            env: BTreeMap::new(),
11912            prompt_delivery: None,
11913            max_turn_secs: None,
11914        }
11915    }
11916
11917    fn ids(xs: &[&str]) -> BTreeSet<String> {
11918        xs.iter().map(|s| (*s).to_owned()).collect()
11919    }
11920
11921    #[test]
11922    fn pick_successor_skips_an_agent_another_seat_holds() {
11923        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
11924        let roster = [spec("a"), spec("b"), spec("c")];
11925        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11926        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11927    }
11928
11929    #[test]
11930    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11931        // The other seat started on c but was handed to d; c is free again.
11932        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11933        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11934        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11935        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11936        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11937    }
11938
11939    #[test]
11940    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11941        let roster = [spec("a"), spec("b")];
11942        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11943        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11944        let carried = ids(&["b"]);
11945        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11946        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11947    }
11948
11949    #[test]
11950    fn pick_successor_returns_none_without_any_untried_successor() {
11951        let roster = [spec("a"), spec("b")];
11952        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11953        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11954        let carried = ids(&["a"]);
11955        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11956    }
11957
11958    #[test]
11959    fn pick_successor_rescue_avoids_another_seats_occupant() {
11960        // b is a carried failure and free; a is held by the other seat.
11961        let roster = [spec("a"), spec("b"), spec("c")];
11962        let carried = ids(&["b", "c"]);
11963        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11964        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11965    }
11966
11967    #[test]
11968    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11969        let roster = [spec("a"), spec("b"), spec("c")];
11970        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11971        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11972    }
11973
11974    #[test]
11975    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11976        let roster = [spec("a"), spec("b"), spec("c")];
11977        let failed = ids(&["a", "b", "c"]);
11978        // Rescue looks at the whole roster, once per id, then runs out.
11979        let mut tried = ids(&["b"]);
11980        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11981        assert_eq!(first.id, "a");
11982        tried.insert(first.id.clone());
11983        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11984        assert_eq!(second.id, "c");
11985        tried.insert(second.id.clone());
11986        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11987    }
11988
11989    #[test]
11990    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11991        let roster = [spec("a"), spec("b")];
11992        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11993        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11994    }
11995
11996    #[test]
11997    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11998        let roster = [spec("a"), spec("b"), spec("c")];
11999        let h = SeatHistory {
12000            failed: ids(&["a"]),
12001            last_ok: Some("b".to_owned()),
12002            last_fail: None,
12003        };
12004        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12005        // A last answerer that left the roster, or later failed, is ignored.
12006        let gone = SeatHistory {
12007            last_ok: Some("zzz".to_owned()),
12008            ..h.clone()
12009        };
12010        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12011        let failed = SeatHistory {
12012            failed: ids(&["a", "b"]),
12013            last_ok: Some("b".to_owned()),
12014            last_fail: None,
12015        };
12016        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12017    }
12018
12019    #[test]
12020    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12021        let first = SeatState::new("review-1", "alpha", 7);
12022        let next = handover_seat("review-1", "gamma", 7);
12023        assert_ne!(first.claude_session, next.claude_session);
12024    }
12025
12026    #[test]
12027    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12028        let mut state = state_with_summary("x", "y");
12029        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12030        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12031        assert_ne!(a.claude_session, b.claude_session);
12032    }
12033
12034    #[test]
12035    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12036        let roster = [spec("a"), spec("b")];
12037        let h = SeatHistory {
12038            failed: ids(&["a", "b"]),
12039            ..SeatHistory::default()
12040        };
12041        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12042        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12043        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12044    }
12045
12046    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12047    // fallback loop depends on to terminate: it must walk forward from the
12048    // seat's own position, never restart at the front of the roster, and it
12049    // must never hand back an id already tried, however many times that id
12050    // happens to appear.
12051
12052    #[test]
12053    fn failure_signature_ignores_numbers_and_paths() {
12054        assert_eq!(
12055            failure_signature("exited with Some(2) and no usable output"),
12056            failure_signature("exited with Some(137) and no usable output")
12057        );
12058        assert_eq!(
12059            failure_signature("process exited with no usable output (exit code 2)"),
12060            failure_signature("process exited with no usable output (exit code 137)")
12061        );
12062        for (old, new) in [
12063            (
12064                "exited with Some(2) and no usable output",
12065                "process exited with no usable output (exit code 2)",
12066            ),
12067            (
12068                "exited with None and no usable output",
12069                "process exited with no usable output (exit code unavailable)",
12070            ),
12071        ] {
12072            let previous = FailClass::Other(failure_signature(old));
12073            let current = FailClass::Other(failure_signature(new));
12074            assert_eq!(previous, current, "resumption retains the failure class");
12075            assert!(!should_hand_over(Some(&previous), &current));
12076        }
12077        assert_eq!(
12078            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12079            failure_signature("cannot open /var/x.txt: denied")
12080        );
12081        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12082    }
12083
12084    #[test]
12085    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12086        let other = FailClass::Other("x".into());
12087        assert!(should_hand_over(None, &FailClass::Quota));
12088        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12089        assert!(should_hand_over(
12090            Some(&FailClass::Timeout),
12091            &FailClass::Timeout
12092        ));
12093        assert!(should_hand_over(None, &other));
12094        assert!(!should_hand_over(Some(&other), &other));
12095        assert!(should_hand_over(
12096            Some(&other),
12097            &FailClass::Other("y".into())
12098        ));
12099        // A quota or timeout in between ends the run of identical failures.
12100        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12101        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12102    }
12103
12104    #[test]
12105    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12106        let a = SeatState::new("judge-1", "alpha", 7);
12107        let b = handover_seat("judge-1", "beta", 7);
12108        assert_ne!(a.claude_session, b.claude_session);
12109        assert_eq!(b.turns, 0);
12110    }
12111
12112    #[test]
12113    fn a_timeout_is_classified_apart_from_other_failures() {
12114        assert_eq!(
12115            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12116            Some(FailClass::Timeout)
12117        );
12118        assert!(matches!(
12119            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12120            Some(FailClass::Other(_))
12121        ));
12122    }
12123
12124    #[test]
12125    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12126        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12127        let tried = BTreeSet::from(["beta".to_owned()]);
12128        // beta sits at index 1; the next candidate is gamma, never alpha —
12129        // which is very likely a different candidate slot's own agent.
12130        let next = next_untried_in_roster(&roster, 1, &tried);
12131        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12132    }
12133
12134    #[test]
12135    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12136        let roster = vec![spec("alpha"), spec("beta")];
12137        let tried = BTreeSet::from(["beta".to_owned()]);
12138        // beta is the roster's last entry: nothing follows it, and alpha —
12139        // earlier in the roster, almost certainly a different candidate
12140        // slot's own agent — must not be reached by wrapping back to it.
12141        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12142    }
12143
12144    #[test]
12145    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12146        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12147        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12148        // beta (index 1) and gamma (index 2, the only entry after it) have
12149        // both been tried; alpha (index 0) never has, but it comes before
12150        // beta's own position, so there is nothing further for this seat.
12151        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12152    }
12153
12154    #[test]
12155    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12156        let roster = vec![spec("a"), spec("a"), spec("b")];
12157        let tried = BTreeSet::from(["a".to_owned()]);
12158        let next = next_untried_in_roster(&roster, 0, &tried);
12159        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12160    }
12161
12162    #[test]
12163    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12164        let roster = vec![spec("a"), spec("b")];
12165        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12166        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12167    }
12168
12169    #[test]
12170    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12171        let dir = tempfile::tempdir().unwrap();
12172        let bay = dir.path().join("ffff");
12173
12174        // Not there yet: nothing to do, nothing to panic on.
12175        remove_if_empty(&bay);
12176        assert!(!bay.exists());
12177
12178        // Something still inside - the winner's worktree, or a stray file -
12179        // keeps the directory standing.
12180        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12181        remove_if_empty(&bay);
12182        assert!(bay.exists(), "non-empty directory must survive");
12183
12184        // Once the last entry is gone, so is the directory itself.
12185        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12186        remove_if_empty(&bay);
12187        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12188    }
12189
12190    // `round_is_clean` is the exact decision this task fixed: a round with a
12191    // seat that never answered must not read the same as a round every seat
12192    // actually reviewed. These are deterministic and process-free by design —
12193    // the equivalent end-to-end check (a real reviewer timing out under a
12194    // live graph run) is a genuine race against wall-clock contention, and a
12195    // spawn slow enough to blow even a generous budget under a loaded test
12196    // run must not turn this specific regression check flaky.
12197
12198    #[test]
12199    fn a_full_panel_that_found_nothing_is_clean() {
12200        assert!(round_is_clean(
12201            0,
12202            true,
12203            2,
12204            2,
12205            0,
12206            IncompleteReviewPolicy::Block
12207        ));
12208    }
12209
12210    #[test]
12211    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12212        assert!(!round_is_clean(
12213            0,
12214            true,
12215            1,
12216            2,
12217            0,
12218            IncompleteReviewPolicy::Block
12219        ));
12220    }
12221
12222    #[test]
12223    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12224        assert!(!round_is_clean(
12225            1,
12226            true,
12227            1,
12228            2,
12229            0,
12230            IncompleteReviewPolicy::Warn
12231        ));
12232    }
12233
12234    #[test]
12235    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12236        assert!(round_is_clean(
12237            0,
12238            true,
12239            1,
12240            2,
12241            0,
12242            IncompleteReviewPolicy::Warn
12243        ));
12244    }
12245
12246    #[test]
12247    fn a_full_panel_with_an_open_finding_is_not_clean() {
12248        assert!(!round_is_clean(
12249            1,
12250            true,
12251            2,
12252            2,
12253            0,
12254            IncompleteReviewPolicy::Block
12255        ));
12256    }
12257
12258    #[test]
12259    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12260        assert!(!round_is_clean(
12261            0,
12262            false,
12263            2,
12264            2,
12265            0,
12266            IncompleteReviewPolicy::Block
12267        ));
12268    }
12269
12270    // The stall this task closes: under the default `block` policy, a seat
12271    // missing only because it was rate limited must not force a wait for a
12272    // session limit that will not lift by the next round. `round_is_clean`
12273    // is where that quorum carve-out lives; the review loop around it never
12274    // changes what a reviewer's vote or a finding's severity means.
12275
12276    #[test]
12277    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12278        // 1 of 2 answered, and the one missing was quota'd — the exact
12279        // "review-2 rate limited (quota)" shape from the field report.
12280        assert!(round_is_clean(
12281            0,
12282            true,
12283            1,
12284            2,
12285            1,
12286            IncompleteReviewPolicy::Block
12287        ));
12288    }
12289
12290    #[test]
12291    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12292        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12293        // not a quota loss (`quota_missing` stays 0) — worth another try.
12294        assert!(!round_is_clean(
12295            0,
12296            true,
12297            1,
12298            2,
12299            0,
12300            IncompleteReviewPolicy::Block
12301        ));
12302    }
12303
12304    #[test]
12305    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12306        assert!(!round_is_clean(
12307            1,
12308            true,
12309            1,
12310            2,
12311            1,
12312            IncompleteReviewPolicy::Block
12313        ));
12314        assert!(!round_is_clean(
12315            0,
12316            false,
12317            1,
12318            2,
12319            1,
12320            IncompleteReviewPolicy::Block
12321        ));
12322    }
12323
12324    #[test]
12325    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12326        // Every seat quota'd, nobody answered: there is no panel to decide
12327        // on, so this must fall through to the existing block-and-retry
12328        // fallback rather than call an unreviewed patch clean.
12329        assert!(!round_is_clean(
12330            0,
12331            true,
12332            0,
12333            2,
12334            2,
12335            IncompleteReviewPolicy::Block
12336        ));
12337    }
12338
12339    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12340        CommandOutcome {
12341            command: "test".to_owned(),
12342            code,
12343            output_tail: String::new(),
12344            duration_ms: 0,
12345            resource_blocked,
12346        }
12347    }
12348
12349    #[test]
12350    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12351        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12352        assert!(
12353            !verify_inconclusive(&[outcome(Some(1), false)]),
12354            "an ordinary failure is still evidence about the patch"
12355        );
12356        assert!(verify_inconclusive(&[outcome(None, true)]));
12357        assert!(
12358            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12359            "one inconclusive outcome taints the whole batch"
12360        );
12361        assert!(!verify_inconclusive(&[]));
12362    }
12363
12364    #[tokio::test]
12365    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12366        // Alive for the first two checks, then dead - confirms the loop
12367        // actually re-polls rather than deciding once and sleeping out the
12368        // ceiling regardless.
12369        let calls = std::sync::atomic::AtomicUsize::new(0);
12370        let started = Instant::now();
12371        wait_for_pids_with(
12372            &[123],
12373            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12374            Duration::from_millis(5),
12375            Duration::from_secs(5),
12376        )
12377        .await;
12378        assert!(
12379            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12380            "must keep checking rather than deciding on the first answer"
12381        );
12382        assert!(
12383            started.elapsed() < Duration::from_secs(1),
12384            "must return the moment it is confirmed dead, not wait out the ceiling"
12385        );
12386    }
12387
12388    #[tokio::test]
12389    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12390        let started = Instant::now();
12391        wait_for_pids_with(
12392            &[123],
12393            |_| true, // never reports dead
12394            Duration::from_millis(5),
12395            Duration::from_millis(30),
12396        )
12397        .await;
12398        let elapsed = started.elapsed();
12399        assert!(
12400            elapsed >= Duration::from_millis(30),
12401            "must not give up before its own ceiling: {elapsed:?}"
12402        );
12403        assert!(
12404            elapsed < Duration::from_secs(1),
12405            "must not wait past its own ceiling either: {elapsed:?}"
12406        );
12407    }
12408
12409    #[tokio::test]
12410    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12411        let started = Instant::now();
12412        wait_for_pids_with(
12413            &[],
12414            |_| true,
12415            Duration::from_secs(5),
12416            Duration::from_secs(5),
12417        )
12418        .await;
12419        assert!(
12420            started.elapsed() < Duration::from_millis(200),
12421            "an empty pid list has nothing to confirm"
12422        );
12423    }
12424
12425    // `review_conclusion` is the exact decision the review hand-off task
12426    // fixed: a round budget spent (or a tree that stopped moving) must not
12427    // collapse into `Blocked` regardless of what verification actually
12428    // said. Deterministic and process-free for the same reason the
12429    // `round_is_clean` family above is.
12430    fn review_round(
12431        clean: bool,
12432        blocking: usize,
12433        answered: usize,
12434        expected: usize,
12435        progressed: bool,
12436        e2e_ok: bool,
12437    ) -> ReviewRound {
12438        ReviewRound {
12439            round: 1,
12440            head: "h".to_owned(),
12441            verified_head: None,
12442            verified_at: None,
12443            reviews: Vec::new(),
12444            e2e: vec![CommandOutcome {
12445                command: "test".to_owned(),
12446                code: Some(if e2e_ok { 0 } else { 1 }),
12447                output_tail: String::new(),
12448                duration_ms: 0,
12449                resource_blocked: false,
12450            }],
12451            verify_retried: false,
12452            e2e_deferred: false,
12453            e2e_defer_reason: None,
12454            fix: None,
12455            blocking,
12456            answered,
12457            expected,
12458            clean,
12459            progressed,
12460            vote_split: false,
12461            reconsideration: Vec::new(),
12462            verdict: None,
12463        }
12464    }
12465
12466    #[test]
12467    fn review_conclusion_is_none_when_nothing_has_run() {
12468        assert_eq!(review_conclusion(&[], 3), None);
12469    }
12470
12471    #[test]
12472    fn review_conclusion_is_none_while_rounds_remain() {
12473        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12474        assert_eq!(review_conclusion(&rounds, 3), None);
12475    }
12476
12477    #[test]
12478    fn review_conclusion_is_gating_once_a_round_is_clean() {
12479        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12480        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12481    }
12482
12483    #[test]
12484    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12485        let rounds = vec![
12486            review_round(false, 1, 2, 2, true, true),
12487            review_round(false, 1, 2, 2, true, true),
12488        ];
12489        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12490    }
12491
12492    #[test]
12493    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12494        let rounds = vec![
12495            review_round(false, 1, 2, 2, true, true),
12496            review_round(false, 1, 2, 2, true, false),
12497        ];
12498        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12499    }
12500
12501    #[test]
12502    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12503        // Magi never got a command to run against this round's own head — a
12504        // resource-blocked attempt, not a red one — so this must never
12505        // settle on `Blocked` the way a genuine e2e failure would. `None`
12506        // here is what tells `Runner::review_loop` to retry the check
12507        // itself rather than trust this cheap recomputation with a verdict
12508        // it cannot actually produce.
12509        let mut blocked = review_round(false, 1, 2, 2, true, false);
12510        blocked.e2e[0].resource_blocked = true;
12511        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12512        assert_eq!(review_conclusion(&rounds, 2), None);
12513    }
12514
12515    #[test]
12516    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12517        // Missing input, not a verified tree — never a hand-off candidate.
12518        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12519        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12520    }
12521
12522    #[test]
12523    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12524        let rounds = vec![
12525            review_round(false, 1, 2, 2, false, true),
12526            review_round(false, 1, 2, 2, false, true),
12527        ];
12528        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12529    }
12530
12531    fn secs(n: u64) -> Duration {
12532        Duration::from_secs(n)
12533    }
12534
12535    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12536    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12537    #[tokio::test]
12538    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12539        ask_test_home();
12540        let tmp = tempfile::tempdir().unwrap();
12541        let repo = tmp.path().join("repo");
12542        std::fs::create_dir_all(&repo).unwrap();
12543        init_repo(&repo);
12544        let g = |args: &[&str]| {
12545            let out = std::process::Command::new("git")
12546                .args(args)
12547                .current_dir(&repo)
12548                .quiet()
12549                .output()
12550                .unwrap();
12551            assert!(
12552                out.status.success(),
12553                "{}",
12554                String::from_utf8_lossy(&out.stderr)
12555            );
12556        };
12557        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12558        g(&["checkout", "-b", "historical"]);
12559        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12560        g(&["add", "."]);
12561        g(&["commit", "-m", "old alternative"]);
12562        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12563        g(&["checkout", "-b", "magi/task-selected", &base]);
12564        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12565        g(&["add", "."]);
12566        g(&["commit", "-m", "measurement infrastructure"]);
12567        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12568        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12569        g(&["add", "."]);
12570        g(&["commit", "-m", "dependent implementation"]);
12571        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12572        g(&["checkout", "main"]);
12573        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12574        g(&["add", "."]);
12575        g(&["commit", "-m", "unrelated base change"]);
12576        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12577        let origin = tmp.path().join("origin.git");
12578        g(&[
12579            "clone",
12580            "--bare",
12581            repo.to_str().unwrap(),
12582            origin.to_str().unwrap(),
12583        ]);
12584        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12585        g(&["fetch", "origin"]);
12586        let selection = crate::pr_recovery::Selection {
12587            url: "https://example.test/project/pull/1".into(),
12588            branch: "magi/task-selected".into(),
12589            head: head.clone(),
12590            source_head: None,
12591            source_run: None,
12592            source_proof: None,
12593            disposition: Some(crate::pr_recovery::Disposition::Review),
12594            instruction: "preserve measured behavior".into(),
12595            approved_at: jiff::Timestamp::now(),
12596        };
12597        let mut template = runner_at(RunStatus::Reviewing);
12598        template.state.repo = repo.clone();
12599        template.state.base_commit = current.clone();
12600        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12601        template.state.origin = Some(crate::run::Origin::queue("selected"));
12602        template.state.pr_head = Some(selection.branch.clone());
12603        template.state.pr_recovery = Some(selection.clone());
12604        let mut runner =
12605            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12606                .await
12607                .unwrap();
12608        let wt = runner.state.winner().unwrap().worktree.clone();
12609        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12610        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12611        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12612        assert!(runner.state.seeds.is_empty());
12613        assert_eq!(runner.state.instruction, "preserve measured behavior");
12614        runner.sync_to_base().await.unwrap();
12615        assert_eq!(
12616            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12617            "prerequisite"
12618        );
12619        assert_eq!(
12620            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12621            "measured implementation"
12622        );
12623        assert_eq!(
12624            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12625            "current main change"
12626        );
12627        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12628        assert_eq!(
12629            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12630            head
12631        );
12632
12633        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12634        let mut partial = runner_at(RunStatus::Reviewing);
12635        partial.state.repo = repo.clone();
12636        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12637        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12638            run: Some("exit 1".into()),
12639            ..Default::default()
12640        }];
12641        let selection = runner.state.pr_recovery.clone().unwrap();
12642        partial.state.pr_recovery = Some(selection.clone());
12643        partial.state.pr_head = Some(selection.branch.clone());
12644        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12645        let id = partial.state.id.clone();
12646        let error = match Runner::open_selected(
12647            &repo,
12648            partial.state,
12649            partial.roles,
12650            1,
12651            runner.state.base_commit.clone(),
12652            selection,
12653        )
12654        .await
12655        {
12656            Ok(_) => panic!("setup must fail"),
12657            Err(e) => e,
12658        };
12659        assert_eq!(
12660            error
12661                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12662                .unwrap()
12663                .run,
12664            id
12665        );
12666        let failed = RunState::load(&id).unwrap();
12667        assert_eq!(failed.status, RunStatus::Blocked);
12668        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12669        assert!(Runner::resume(&id).is_err());
12670        runner.state.status = RunStatus::Ready;
12671        runner.state.driver_exited = true;
12672        runner.state.save().unwrap();
12673        let mut retry = runner_at(RunStatus::Reviewing);
12674        retry.state.repo = repo.clone();
12675        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12676        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12677        reselected.approved_at = jiff::Timestamp::now();
12678        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12679        retry.state.pr_head = Some(reselected.branch.clone());
12680        retry.state.pr_recovery = Some(reselected.clone());
12681        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12682            .await
12683            .unwrap();
12684        let restarted = Runner::open_selected(
12685            &repo,
12686            retry.state,
12687            retry.roles,
12688            1,
12689            runner.state.base_commit.clone(),
12690            reselected,
12691        )
12692        .await
12693        .unwrap();
12694        assert_eq!(
12695            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12696                .await
12697                .unwrap(),
12698            head
12699        );
12700        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12701        assert_eq!(
12702            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12703            head
12704        );
12705    }
12706
12707    #[tokio::test]
12708    #[cfg(unix)]
12709    async fn fallback_and_interrupted_restoration_require_coordination() {
12710        use std::os::fd::AsRawFd as _;
12711        ask_test_home();
12712        let tmp = tempfile::tempdir().unwrap();
12713        let mut runner = registered_runner(tmp.path()).await;
12714        runner.state.config.graph.timeout_resource_wait = 1;
12715        let winner = runner.state.candidates[0].clone();
12716        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12717        let leftover = winner.worktree.join("leftover");
12718        std::fs::write(&leftover, "untrusted edits").unwrap();
12719        let next = tmp.path().join("next-fixer");
12720        let mut failed = spec("first");
12721        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12722        let mut successor = spec("second");
12723        successor.command = vec![
12724            "sh".into(),
12725            "-c".into(),
12726            format!("touch '{}'; echo done", next.display()),
12727        ];
12728        let attempts = vec![
12729            (failed.clone(), "first".into()),
12730            (successor.clone(), "second".into()),
12731        ];
12732        let lock_path = runner.state.repo.join(".git/updater.lock");
12733        let external = std::fs::File::create(&lock_path).unwrap();
12734        assert_eq!(
12735            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12736            0
12737        );
12738        let artifacts = tmp.path().join("artifacts");
12739        let build = |spec, seat| SeatJob {
12740            spec,
12741            seat,
12742            cwd: winner.worktree.clone(),
12743            prompt: "fixture".into(),
12744            timeout: Duration::from_secs(10),
12745            allow_write: true,
12746            sessions: false,
12747            artifacts: artifacts.clone(),
12748            stem: "fix".into(),
12749            handover: None,
12750        };
12751        let (_, _, out) = runner
12752            .ask_fixer_among(
12753                attempts.clone(),
12754                "graft",
12755                None,
12756                Some((&winner.worktree, &head)),
12757                build,
12758            )
12759            .await;
12760        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12761        assert!(leftover.exists(), "contention must prevent clean");
12762        assert!(!next.exists());
12763        assert!(runner.state.handovers.is_empty());
12764        // Early interrupted-repair rollback uses the same protected route.
12765        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12766            from_run: "earlier".into(),
12767            generation: 1,
12768            mode: crate::recovery::Mode::VerifyRepair,
12769            reasons: Vec::new(),
12770            reviewed_head: None,
12771            rounds: Vec::new(),
12772            open_findings: Vec::new(),
12773            evidence: None,
12774            criteria_inherited: true,
12775            repair: crate::recovery::Repair::Started {
12776                before: head.clone(),
12777            },
12778        });
12779        assert!(
12780            runner
12781                .recovery_repair(&winner)
12782                .await
12783                .unwrap_err()
12784                .is::<crate::repo_lock::ResourceBlocked>()
12785        );
12786        assert!(matches!(
12787            runner.state.recovery.as_ref().unwrap().repair,
12788            crate::recovery::Repair::Started { .. }
12789        ));
12790        assert!(leftover.exists());
12791        drop(external);
12792        runner.recovery_repair(&winner).await.unwrap();
12793        assert!(!leftover.exists());
12794        assert!(matches!(
12795            runner.state.recovery.as_ref().unwrap().repair,
12796            crate::recovery::Repair::Done { .. }
12797        ));
12798        // A hard lock-open failure also leaves HEAD and untracked work intact.
12799        std::fs::remove_file(&lock_path).unwrap();
12800        std::fs::create_dir(&lock_path).unwrap();
12801        std::fs::write(&leftover, "untrusted edits").unwrap();
12802        let (_, _, out) = runner
12803            .ask_fixer_among(
12804                attempts,
12805                "graft",
12806                None,
12807                Some((&winner.worktree, &head)),
12808                build,
12809            )
12810            .await;
12811        assert!(matches!(out, AgentOutcome::Failed(_)));
12812        assert!(leftover.exists());
12813        assert!(!next.exists());
12814        assert_eq!(
12815            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12816            head
12817        );
12818    }
12819
12820    #[cfg(unix)]
12821    async fn registered_runner(root: &Path) -> Runner {
12822        ask_test_home();
12823        let repo = root.join("repo");
12824        std::fs::create_dir_all(&repo).unwrap();
12825        init_repo(&repo);
12826        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12827        let mut runner = runner_at(RunStatus::Prep);
12828        runner.state.repo = repo.clone();
12829        runner.state.config.coordination = crate::config::test_coordination();
12830        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12831        runner.state.config.graph.worktree_root = Some(root.join("wt"));
12832        runner.state.config.graph.advise = false;
12833        runner.roles.implementers = vec![spec("implement")];
12834        runner.roles.judges = vec![spec("judge")];
12835        runner.prep().await.unwrap();
12836        runner
12837    }
12838
12839    #[tokio::test]
12840    #[cfg(unix)]
12841    async fn parked_implementer_persists_completed_work() {
12842        for registered in [false, true] {
12843            let tmp = tempfile::tempdir().unwrap();
12844            let mut runner = registered_runner(tmp.path()).await;
12845            if !registered {
12846                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12847            }
12848            let marker = tmp.path().join("started");
12849            let release = tmp.path().join("release");
12850            let mut implementer = spec("implement");
12851            implementer.command = vec![
12852                "sh".into(),
12853                "-c".into(),
12854                format!(
12855                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12856                    marker.display(),
12857                    release.display()
12858                ),
12859            ];
12860            runner.roles.implementers = vec![implementer.clone()];
12861            runner.state.config.agents = vec![implementer];
12862            let pause = runner.pause.clone();
12863            let (result, ()) = tokio::join!(runner.implement(), async {
12864                tokio::time::timeout(Duration::from_secs(10), async {
12865                    while !marker.exists() {
12866                        tokio::time::sleep(Duration::from_millis(10)).await;
12867                    }
12868                })
12869                .await
12870                .unwrap();
12871                pause.park();
12872                std::fs::write(&release, "go").unwrap();
12873            });
12874            result.unwrap();
12875            let candidate = &runner.state.candidates[0];
12876            assert!(candidate.commits > 0, "{candidate:?}");
12877            assert!(candidate.summary.contains("saved-result"));
12878            assert_eq!(candidate.files, 1);
12879            let before = candidate.commits;
12880            runner.implement().await.unwrap();
12881            assert_eq!(runner.state.candidates[0].commits, before);
12882        }
12883    }
12884
12885    #[tokio::test]
12886    #[cfg(unix)]
12887    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12888        let tmp = tempfile::tempdir().unwrap();
12889        let mut runner = registered_runner(tmp.path()).await;
12890        let winner = runner.state.candidates[0].clone();
12891        let cache = tmp.path().join("cache");
12892        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12893        runner.state.config.graph.timeout_resource_wait = 0;
12894        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12895        let crate::cache::AcquireOutcome::Acquired(held) =
12896            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12897        else {
12898            panic!("cache must be free")
12899        };
12900        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12901            from_run: "earlier".into(),
12902            generation: 1,
12903            mode: crate::recovery::Mode::VerifyRepair,
12904            reasons: Vec::new(),
12905            reviewed_head: None,
12906            rounds: Vec::new(),
12907            open_findings: Vec::new(),
12908            evidence: None,
12909            criteria_inherited: true,
12910            repair: crate::recovery::Repair::NotStarted,
12911        });
12912        runner.recovery_repair(&winner).await.unwrap();
12913        assert!(matches!(
12914            runner.state.recovery.as_ref().unwrap().repair,
12915            crate::recovery::Repair::WaitingForRestore { .. }
12916        ));
12917        drop(held);
12918        runner.state.resource_block = None;
12919        runner.state.config.graph.timeout_resource_wait = 1;
12920        runner.recovery_repair(&winner).await.unwrap();
12921        assert!(matches!(
12922            runner.state.recovery.as_ref().unwrap().repair,
12923            crate::recovery::Repair::Done { .. }
12924        ));
12925        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12926    }
12927
12928    #[tokio::test]
12929    #[cfg(unix)]
12930    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12931        use std::os::fd::AsRawFd as _;
12932        for graft in [false, true] {
12933            let tmp = tempfile::tempdir().unwrap();
12934            let mut runner = registered_runner(tmp.path()).await;
12935            runner.state.config.graph.timeout_resource_wait = 1;
12936            let winner = runner.state.candidates[0].clone();
12937            let marker = tmp.path().join("fixer-started");
12938            let mut fixer = spec("fixer");
12939            fixer.command = vec![
12940                "sh".into(),
12941                "-c".into(),
12942                format!("echo started >> '{}'; echo done", marker.display()),
12943            ];
12944            runner.roles.fixer = Some(vec![fixer]);
12945            if graft {
12946                let mut loser = winner.clone();
12947                loser.label = 'B';
12948                loser.index = 1;
12949                runner.state.candidates.push(loser);
12950                runner.state.config.graph.graft_runner_up = true;
12951                runner.state.tally = Some(Tally {
12952                    first_choice: BTreeMap::new(),
12953                    borda: BTreeMap::new(),
12954                    winner: winner.label,
12955                    rankings: 1,
12956                    unanimous_initial: true,
12957                    deliberated: false,
12958                    changed_votes: 0,
12959                    unanimous_final: true,
12960                    tie_break: None,
12961                    judges: 1,
12962                    present: 1,
12963                    quorum: 1,
12964                    met_quorum: true,
12965                    uncontested: None,
12966                });
12967                runner.state.judgements.push(crate::run::Judgement {
12968                    judge: 1,
12969                    seat: "judge-1".into(),
12970                    agent: "judge".into(),
12971                    ranking: vec![winner.label, 'B'],
12972                    reasons: BTreeMap::new(),
12973                    confidence: None,
12974                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12975                    order: vec![0, 1],
12976                    failed: None,
12977                    duration_ms: 0,
12978                });
12979            } else {
12980                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12981                    from_run: "earlier".into(),
12982                    generation: 1,
12983                    mode: crate::recovery::Mode::VerifyRepair,
12984                    reasons: Vec::new(),
12985                    reviewed_head: None,
12986                    rounds: Vec::new(),
12987                    open_findings: Vec::new(),
12988                    evidence: None,
12989                    criteria_inherited: true,
12990                    repair: crate::recovery::Repair::NotStarted,
12991                });
12992            }
12993            let reader = crate::repo_lock::SeatLease::for_checkout(
12994                &winner.worktree,
12995                false,
12996                crate::config::test_coordination().as_ref(),
12997            )
12998            .await
12999            .unwrap();
13000            let pause = runner.pause.clone();
13001            let (result, ()) = tokio::join!(
13002                async {
13003                    if graft {
13004                        runner.graft().await
13005                    } else {
13006                        runner.recovery_repair(&winner).await
13007                    }
13008                },
13009                async {
13010                    tokio::time::sleep(Duration::from_millis(100)).await;
13011                    pause.park();
13012                }
13013            );
13014            result.unwrap();
13015            assert!(!marker.exists(), "a blocked seat must not spawn");
13016            // Exercise the persisted state, as a binary replacement/resume does.
13017            runner.state = RunState::load(&runner.state.id).unwrap();
13018            if graft {
13019                assert_eq!(
13020                    runner.state.graft.as_ref().unwrap().outcome,
13021                    GraftOutcome::WaitingForRestore
13022                );
13023            } else {
13024                assert!(matches!(
13025                    runner.state.recovery.as_ref().unwrap().repair,
13026                    crate::recovery::Repair::WaitingForRestore { .. }
13027                ));
13028            }
13029            drop(reader);
13030            runner.pause = Pause::new();
13031            runner.state.resource_block = None;
13032            // Deferred restoration must still respect an external lock owner.
13033            let external =
13034                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13035            assert_eq!(
13036                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13037                0
13038            );
13039            let leftover = winner.worktree.join("leftover");
13040            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13041            let result = if graft {
13042                runner.graft().await
13043            } else {
13044                runner.recovery_repair(&winner).await
13045            };
13046            assert!(
13047                result
13048                    .unwrap_err()
13049                    .is::<crate::repo_lock::ResourceBlocked>()
13050            );
13051            assert!(leftover.exists());
13052            assert!(!marker.exists());
13053            drop(external);
13054            runner.state.resource_block = None;
13055            let result = if graft {
13056                runner.graft().await
13057            } else {
13058                runner.recovery_repair(&winner).await
13059            };
13060            result.unwrap();
13061            assert!(!leftover.exists(), "restore must precede retry");
13062            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13063            if graft {
13064                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13065            } else {
13066                assert!(matches!(
13067                    runner.state.recovery.as_ref().unwrap().repair,
13068                    crate::recovery::Repair::Done { .. }
13069                ));
13070            }
13071        }
13072    }
13073
13074    #[tokio::test]
13075    #[cfg(unix)]
13076    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13077        use std::os::fd::AsRawFd as _;
13078        let tmp = tempfile::tempdir().unwrap();
13079        let mut runner = registered_runner(tmp.path()).await;
13080        runner.state.config.graph.timeout_resource_wait = 1;
13081        let winner = runner.state.candidates[0].clone();
13082        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13083        runner.state.tally = Some(Tally {
13084            first_choice: BTreeMap::new(),
13085            borda: BTreeMap::new(),
13086            winner: winner.label,
13087            rankings: 0,
13088            unanimous_initial: false,
13089            deliberated: false,
13090            changed_votes: 0,
13091            unanimous_final: false,
13092            tie_break: None,
13093            judges: 0,
13094            present: 0,
13095            quorum: 0,
13096            met_quorum: true,
13097            uncontested: Some("fixture".into()),
13098        });
13099        runner.state.graft = Some(GraftRecord {
13100            pre: pre.clone(),
13101            post: None,
13102            lines: None,
13103            limit: None,
13104            agent: None,
13105            outcome: GraftOutcome::Running,
13106            requested: Vec::new(),
13107            adoption: None,
13108            adoption_error: None,
13109        });
13110        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13111        git::commit_all(&winner.worktree, "unfinished graft")
13112            .await
13113            .unwrap();
13114        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13115        let leftover = winner.worktree.join("leftover");
13116        std::fs::write(&leftover, "untrusted edits").unwrap();
13117        let path = runner.state.repo.join(".git/updater.lock");
13118        let external = std::fs::File::create(&path).unwrap();
13119        assert_eq!(
13120            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13121            0
13122        );
13123        assert!(
13124            runner
13125                .graft()
13126                .await
13127                .unwrap_err()
13128                .is::<crate::repo_lock::ResourceBlocked>()
13129        );
13130        assert!(matches!(
13131            runner.state.graft.as_ref().unwrap().outcome,
13132            GraftOutcome::Running
13133        ));
13134        assert_eq!(
13135            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13136            unfinished
13137        );
13138        assert!(leftover.exists());
13139        drop(external);
13140        std::fs::remove_file(&path).unwrap();
13141        std::fs::create_dir(&path).unwrap();
13142        assert!(runner.graft().await.is_err());
13143        assert!(matches!(
13144            runner.state.graft.as_ref().unwrap().outcome,
13145            GraftOutcome::Running
13146        ));
13147        assert_eq!(
13148            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13149            unfinished
13150        );
13151        assert!(leftover.exists());
13152        std::fs::remove_dir(&path).unwrap();
13153        runner.graft().await.unwrap();
13154        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13155        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13156        assert!(!leftover.exists());
13157    }
13158
13159    #[tokio::test]
13160    #[cfg(unix)]
13161    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13162        use std::os::fd::AsRawFd as _;
13163        let tmp = tempfile::tempdir().unwrap();
13164        let mut runner = registered_runner(tmp.path()).await;
13165        runner.state.prep_complete = Some(false);
13166        runner.state.config.graph.timeout_resource_wait = 1;
13167        runner.state.save().unwrap();
13168        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13169        assert_eq!(
13170            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13171            0
13172        );
13173        runner.execute().await.unwrap();
13174        let disk = RunState::load(&runner.state.id).unwrap();
13175        assert!(disk.driver_exited);
13176        assert_eq!(disk.driver_pid, None);
13177        assert_eq!(runner.state.status, RunStatus::Blocked);
13178        assert_eq!(
13179            runner.state.resource_block.as_ref().unwrap().kind,
13180            "repository coordination"
13181        );
13182        assert!(runner.state.prep_failure.is_none());
13183        assert!(runner.state.seats.is_empty());
13184        runner.pause.park();
13185        // Include headroom for Git subprocesses under parallel test load;
13186        // the repository contention deadline itself remains one second.
13187        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13188            .await
13189            .unwrap()
13190            .unwrap();
13191        assert!(runner.state.parked);
13192        assert!(runner.state.prep_failure.is_none());
13193        let disk = RunState::load(&runner.state.id).unwrap();
13194        assert!(disk.driver_exited);
13195        assert_eq!(disk.driver_pid, None);
13196    }
13197
13198    #[tokio::test]
13199    #[cfg(unix)]
13200    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13201        use std::os::fd::AsRawFd as _;
13202        let tmp = tempfile::tempdir().unwrap();
13203        let mut runner = registered_runner(tmp.path()).await;
13204        runner.state.prep_complete = Some(false);
13205        runner.state.config.graph.timeout_resource_wait = 30;
13206        runner.state.save().unwrap();
13207        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13208        assert_eq!(
13209            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13210            0
13211        );
13212        assert!(
13213            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13214                .await
13215                .is_err()
13216        );
13217        let disk = RunState::load(&runner.state.id).unwrap();
13218        assert!(disk.driver_exited);
13219        assert_eq!(disk.driver_pid, None);
13220        assert!(disk.status.resumable());
13221        assert!(runner.state.driver_exited);
13222        assert_eq!(runner.state.driver_pid, None);
13223        runner.state.save().unwrap();
13224        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13225    }
13226
13227    #[tokio::test]
13228    #[cfg(unix)]
13229    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13230        let tmp = tempfile::tempdir().unwrap();
13231        let mut runner = registered_runner(tmp.path()).await;
13232        let winner = runner.state.candidates[0].clone();
13233        let cache = tmp.path().join("cache");
13234        let started = tmp.path().join("pre-gate-started");
13235        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13236        runner.state.config.verify.gate =
13237            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13238        let home = crate::run::home();
13239        let owner =
13240            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13241        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13242            crate::cache::AcquireOutcome::Acquired(g) => g,
13243            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13244        };
13245        let mut gate = Box::pin(runner.run_gate(&winner));
13246        assert!(
13247            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13248                .await
13249                .is_err()
13250        );
13251        assert!(!started.exists(), "pre-gate must wait for the cache too");
13252        // Simulate the wave that already owns the cache entering invoke.
13253        // It must obtain the editor reservation and finish without waiting
13254        // for the gate that is queued behind its cache.
13255        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13256        let editor = crate::repo_lock::SeatLease::checkout_with(
13257            &winner.worktree,
13258            true,
13259            crate::config::test_coordination().as_ref(),
13260            &wait,
13261        )
13262        .await
13263        .unwrap();
13264        drop(editor);
13265        guard.release();
13266        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13267            .await
13268            .unwrap()
13269            .unwrap();
13270        drop(gate);
13271        assert!(outcomes.iter().all(CommandOutcome::ok));
13272        assert!(started.exists());
13273    }
13274
13275    #[tokio::test]
13276    #[cfg(unix)]
13277    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13278        use std::os::fd::AsRawFd as _;
13279
13280        ask_test_home();
13281        let tmp = tempfile::tempdir().unwrap();
13282        let repo = tmp.path().join("repo");
13283        std::fs::create_dir_all(&repo).unwrap();
13284        init_repo(&repo);
13285        let common = repo.join(".git");
13286        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13287        let mut runner = runner_at(RunStatus::Prep);
13288        runner.state.repo = repo.clone();
13289        runner.state.config.coordination = crate::config::test_coordination();
13290        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13291        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13292        runner.state.config.graph.advise = false;
13293        runner.roles.implementers = vec![spec("implement")];
13294        runner.roles.judges = vec![spec("judge")];
13295        runner.prep().await.unwrap();
13296        let winner = runner.state.candidates[0].clone();
13297        // Emulate edits already left by a completed seat. The fixture CLI
13298        // only reports completion; it performs no repository operations.
13299        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13300        let completed = tmp.path().join("seat-completed");
13301        runner.roles.implementers[0].command = vec![
13302            "sh".into(),
13303            "-c".into(),
13304            format!("touch '{}'; echo completed", completed.display()),
13305        ];
13306        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13307        assert_eq!(
13308            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13309            0
13310        );
13311        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13312        let mut implement = Box::pin(runner.implement());
13313        tokio::select! {
13314            result = &mut implement => panic!("rescue must wait: {result:?}"),
13315            result = tokio::time::timeout(Duration::from_secs(10), async {
13316                while !completed.exists() {
13317                    tokio::time::sleep(Duration::from_millis(20)).await;
13318                }
13319            }) => result.unwrap(),
13320        }
13321        assert!(
13322            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13323                .await
13324                .is_err()
13325        );
13326        assert_eq!(
13327            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13328            before
13329        );
13330        assert_eq!(
13331            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13332            0
13333        );
13334        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13335            .await
13336            .unwrap()
13337            .unwrap();
13338        drop(implement);
13339        assert_ne!(
13340            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13341            before
13342        );
13343        assert!(git::is_clean(&winner.worktree).await.unwrap());
13344
13345        let started = tmp.path().join("pre-gate-started");
13346        let proceed = tmp.path().join("pre-gate-proceed");
13347        let checked = tmp.path().join("gate-ran");
13348        runner.state.config.verify.pre_gate = vec![format!(
13349            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13350            started.display(),
13351            proceed.display()
13352        )];
13353        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13354        assert_eq!(
13355            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13356            0
13357        );
13358        let mut gate = Box::pin(runner.run_gate(&winner));
13359        assert!(
13360            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13361                .await
13362                .is_err()
13363        );
13364        assert!(!started.exists());
13365        assert!(!checked.exists());
13366        assert!(git::is_clean(&winner.worktree).await.unwrap());
13367        assert_eq!(
13368            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13369            0
13370        );
13371        tokio::select! {
13372            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13373            result = tokio::time::timeout(Duration::from_secs(10), async {
13374                while !started.exists() {
13375                    tokio::time::sleep(Duration::from_millis(20)).await;
13376                }
13377            }) => result.unwrap(),
13378        }
13379        assert_eq!(
13380            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13381            -1,
13382            "mechanical edits must hold the real lock"
13383        );
13384        std::fs::write(proceed, "ready").unwrap();
13385        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13386            .await
13387            .unwrap()
13388            .unwrap();
13389        drop(gate);
13390        assert!(outcomes.iter().all(CommandOutcome::ok));
13391        assert!(checked.exists());
13392        assert!(runner.state.pre_gate_commit.is_some());
13393        assert!(git::is_clean(&winner.worktree).await.unwrap());
13394        assert_eq!(
13395            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13396            0,
13397            "gate completion must release coordination"
13398        );
13399        drop(outsider);
13400        std::fs::remove_file(common.join("updater.lock")).unwrap();
13401        std::fs::create_dir(common.join("updater.lock")).unwrap();
13402        std::fs::remove_file(&started).unwrap();
13403        std::fs::remove_file(&checked).unwrap();
13404        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13405        let error = runner.run_gate(&winner).await.unwrap_err();
13406        assert!(
13407            error
13408                .to_string()
13409                .contains("acquire mandatory repository lock")
13410        );
13411        assert!(
13412            !started.exists(),
13413            "failed acquisition must prevent pre-gate"
13414        );
13415        assert!(!checked.exists(), "failed acquisition must prevent gate");
13416        assert!(git::is_clean(&winner.worktree).await.unwrap());
13417        assert_eq!(
13418            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13419            head
13420        );
13421    }
13422
13423    #[tokio::test]
13424    #[cfg(unix)]
13425    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13426        use std::os::fd::AsRawFd as _;
13427        use std::time::Duration;
13428
13429        ask_test_home();
13430        let tmp = tempfile::tempdir().unwrap();
13431        let repo = tmp.path().join("repo");
13432        std::fs::create_dir_all(&repo).unwrap();
13433        init_repo(&repo);
13434        let common = repo.join(".git");
13435        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13436        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13437        assert_eq!(
13438            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13439            0
13440        );
13441        let mut runner = runner_at(RunStatus::Prep);
13442        runner.state.repo = repo.clone();
13443        runner.state.config.coordination = crate::config::test_coordination();
13444        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13445        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13446        runner.state.config.graph.advise = true;
13447        runner.state.config.graph.advisors = 1;
13448        runner.roles.implementers = vec![spec("implement")];
13449        runner.roles.judges = vec![spec("judge")];
13450        let root = runner.state.worktree_root();
13451        let started = tmp.path().join("setup-started");
13452        let proceed = tmp.path().join("setup-proceed");
13453        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13454            run: Some(format!(
13455                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13456                started.display(),
13457                proceed.display()
13458            )),
13459            ..Default::default()
13460        }];
13461        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13462        let config = git::git(&repo, &["config", "--local", "--list"])
13463            .await
13464            .unwrap();
13465        let mut prep = Box::pin(runner.prep());
13466        assert!(
13467            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13468                .await
13469                .is_err()
13470        );
13471        assert!(
13472            !root.exists(),
13473            "contention must prevent even checkout creation"
13474        );
13475        assert!(!started.exists(), "contention must prevent setup");
13476        assert_eq!(
13477            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13478            branches
13479        );
13480        assert_eq!(
13481            git::git(&repo, &["config", "--local", "--list"])
13482                .await
13483                .unwrap(),
13484            config
13485        );
13486        drop(lock);
13487        tokio::select! {
13488            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13489            result = tokio::time::timeout(Duration::from_secs(10), async {
13490                while !started.exists() {
13491                    tokio::time::sleep(Duration::from_millis(20)).await;
13492                }
13493            }) => result.unwrap(),
13494        }
13495        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13496        assert_eq!(
13497            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13498            -1
13499        );
13500        assert!(
13501            tokio::time::timeout(
13502                Duration::from_millis(100),
13503                crate::repo_lock::SeatLease::for_checkout(
13504                    &repo,
13505                    true,
13506                    crate::config::test_coordination().as_ref()
13507                )
13508            )
13509            .await
13510            .is_err(),
13511            "preparation must also exclude local editors"
13512        );
13513        std::fs::write(proceed, "ready").unwrap();
13514        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13515            .await
13516            .unwrap()
13517            .unwrap();
13518        drop(prep);
13519        assert_eq!(runner.state.prep_complete, Some(true));
13520        assert!(root.join("judge-1").exists());
13521        assert!(root.join("advisor-1").exists());
13522        assert_eq!(
13523            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13524            0
13525        );
13526    }
13527
13528    #[tokio::test]
13529    #[cfg(unix)]
13530    async fn preparation_lock_failure_prevents_repository_changes() {
13531        ask_test_home();
13532        let tmp = tempfile::tempdir().unwrap();
13533        let repo = tmp.path().join("repo");
13534        std::fs::create_dir_all(&repo).unwrap();
13535        init_repo(&repo);
13536        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13537        // A directory at the required lock path makes opening the real lock
13538        // fail, rather than merely exercising an invalid registration.
13539        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13540        let mut runner = runner_at(RunStatus::Prep);
13541        runner.state.repo = repo.clone();
13542        runner.state.config.coordination = crate::config::test_coordination();
13543        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13544        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13545        runner.roles.implementers = vec![spec("implement")];
13546        let error = runner.prep().await.unwrap_err();
13547        assert!(
13548            error
13549                .to_string()
13550                .contains("acquire mandatory repository lock")
13551        );
13552        assert!(!runner.state.worktree_root().exists());
13553        assert!(runner.state.candidates.is_empty());
13554        assert!(!repo.join(".git/config.worktree").exists());
13555        assert_eq!(
13556            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13557            "* main"
13558        );
13559    }
13560
13561    #[tokio::test]
13562    async fn preparation_records_creation_intent_before_git_can_fail() {
13563        ask_test_home();
13564        let tmp = tempfile::tempdir().unwrap();
13565        let repo = tmp.path().join("repo");
13566        std::fs::create_dir_all(&repo).unwrap();
13567        init_repo(&repo);
13568        let mut runner = runner_at(RunStatus::Prep);
13569        runner.state.repo = repo.clone();
13570        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13571        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13572        runner.state.config.blind.commit_msg_hook = false;
13573        runner.roles.implementers = vec![spec("one")];
13574        let label = blind::assign_labels(1, runner.state.seed)[0];
13575        let branch = runner.state.branch_for(label);
13576        // A conflicting branch makes Git refuse creation before a candidate
13577        // can be registered. The durable intent must still identify it.
13578        git::git(&repo, &["branch", &branch]).await.unwrap();
13579        assert!(runner.prep().await.is_err());
13580        let saved = RunState::load(&runner.state.id).unwrap();
13581        assert!(saved.candidates.is_empty());
13582        let event = saved
13583            .events
13584            .iter()
13585            .find(|e| e.node == "prep-intent")
13586            .unwrap();
13587        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13588        assert_eq!(intent["branch"], branch);
13589        assert_eq!(intent["start"], runner.state.base_commit);
13590        assert_eq!(
13591            intent["worktree"],
13592            runner
13593                .state
13594                .worktree_root()
13595                .join(format!("cand-{label}"))
13596                .to_string_lossy()
13597                .as_ref()
13598        );
13599        assert_eq!(intent["picks"], serde_json::json!([]));
13600    }
13601
13602    #[tokio::test]
13603    async fn invalid_pinned_sources_create_no_partial_worktrees() {
13604        ask_test_home();
13605        let tmp = tempfile::tempdir().unwrap();
13606        let repo = tmp.path().join("repo");
13607        std::fs::create_dir_all(&repo).unwrap();
13608        init_repo(&repo);
13609        let mut runner = runner_at(RunStatus::Prep);
13610        runner.state.repo = repo.clone();
13611        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13612        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13613        runner.state.sources.pick_commits.push(refs::Input {
13614            token: "missing".into(),
13615            sha: "0".repeat(40),
13616            provenance: refs::Provenance::ExplicitCli,
13617        });
13618        assert!(runner.prep().await.is_err());
13619        assert!(runner.state.candidates.is_empty());
13620        assert!(!runner.state.worktree_root().exists());
13621    }
13622
13623    #[tokio::test]
13624    async fn prose_and_legacy_inference_never_import_into_candidates() {
13625        ask_test_home();
13626        let tmp = tempfile::tempdir().unwrap();
13627        let repo = tmp.path().join("repo");
13628        std::fs::create_dir_all(&repo).unwrap();
13629        init_repo(&repo);
13630        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13631        git::git(&repo, &["checkout", "-b", "magi/history/A"])
13632            .await
13633            .unwrap();
13634        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13635        git::git(&repo, &["add", "."]).await.unwrap();
13636        git::git(&repo, &["commit", "-m", "historical implementation"])
13637            .await
13638            .unwrap();
13639        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13640        git::git(&repo, &["checkout", "main"]).await.unwrap();
13641        for instruction in [
13642            format!("Example: magi/history/A at {source}"),
13643            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13644            format!("Never import magi/history/A or {source}"),
13645            format!("Conversation answer: I mentioned {source} only as background"),
13646        ] {
13647            let mut runner = runner_at(RunStatus::Prep);
13648            runner.state.repo = repo.clone();
13649            runner.state.base_commit = base.clone();
13650            runner.state.instruction = instruction.clone();
13651            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13652            runner.state.config.blind.commit_msg_hook = false;
13653            runner.state.seeds = vec![refs::Seed {
13654                token: "magi/history/A".into(),
13655                sha: source.clone(),
13656                kind: refs::SeedKind::Unmerged,
13657                branch: true,
13658                detail: String::new(),
13659            }];
13660            runner.roles.implementers = vec![spec("one")];
13661            assert_eq!(seeded_instruction(&runner.state), instruction);
13662            // Model an old record with inferred seeds but no structured input field.
13663            let mut legacy = serde_json::to_value(&runner.state).unwrap();
13664            legacy.as_object_mut().unwrap().remove("sources");
13665            runner.state = serde_json::from_value(legacy).unwrap();
13666            assert_eq!(runner.state.sources, refs::Sources::default());
13667            runner.state.save().unwrap();
13668            runner.state = RunState::load(&runner.state.id).unwrap();
13669            runner.prep().await.unwrap();
13670            for candidate in &runner.state.candidates {
13671                assert_eq!(
13672                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13673                    base
13674                );
13675                assert!(!candidate.worktree.join("unwanted").exists());
13676            }
13677        }
13678    }
13679
13680    #[tokio::test]
13681    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13682        ask_test_home();
13683        let tmp = tempfile::tempdir().unwrap();
13684        let repo = tmp.path().join("repo");
13685        std::fs::create_dir_all(&repo).unwrap();
13686        init_repo(&repo);
13687        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13688        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13689        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13690        git::git(&repo, &["add", "."]).await.unwrap();
13691        git::git(&repo, &["commit", "-m", "required seed"])
13692            .await
13693            .unwrap();
13694        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13695        git::git(&repo, &["checkout", "main"]).await.unwrap();
13696        let mut runner = runner_at(RunStatus::Prep);
13697        runner.state.repo = repo.clone();
13698        runner.state.base_commit = base.clone();
13699        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13700        runner.state.config.graph.advise = false;
13701        runner.state.config.blind.commit_msg_hook = false;
13702        runner.state.sources =
13703            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13704                .await
13705                .unwrap();
13706        runner.roles.implementers = vec![spec("one"), spec("two")];
13707        runner.roles.judges = vec![spec("judge")];
13708        let label = blind::assign_labels(2, runner.state.seed)[0];
13709        let branch = runner.state.branch_for(label);
13710        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13711        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13712            .await
13713            .unwrap();
13714        runner.state.candidates.push(Candidate {
13715            index: 0,
13716            label,
13717            agent: "one".into(),
13718            branch,
13719            worktree,
13720            summary: String::new(),
13721            stat: String::new(),
13722            files: 0,
13723            commits: 0,
13724            empty: false,
13725            failed: None,
13726            verified_noop: None,
13727            duration_ms: 0,
13728            folded: false,
13729        });
13730        runner.state.save().unwrap();
13731        // Reload the checkpoint immediately before its first cherry-pick.
13732        runner.state = RunState::load(&runner.state.id).unwrap();
13733        assert_eq!(runner.state.prep_complete, Some(false));
13734        runner.prep().await.unwrap();
13735        assert_eq!(runner.state.prep_complete, Some(true));
13736        assert_eq!(runner.state.candidates.len(), 2);
13737        for candidate in &runner.state.candidates {
13738            assert_eq!(
13739                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13740                git::tree_of(&repo, &seed).await.unwrap()
13741            );
13742            assert_eq!(
13743                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13744                "required commit"
13745            );
13746        }
13747        assert!(runner.state.worktree_root().join("judge-1").exists());
13748        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13749        runner.prep().await.unwrap();
13750        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13751
13752        runner.state.prep_complete = Some(false);
13753        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13754            run: Some(format!(
13755                "test -f '{}'",
13756                tmp.path().join("dependency-ready").display()
13757            )),
13758            ..Default::default()
13759        }];
13760        assert!(runner.prep().await.is_err());
13761        assert!(runner.state.prep_failure.is_some());
13762        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13763        runner.state = RunState::load(&runner.state.id).unwrap();
13764        assert!(
13765            runner.prep().await.is_err(),
13766            "correcting the cause alone must not auto-retry"
13767        );
13768        let failed_at = runner.state.events.last().unwrap().at;
13769        assert!(
13770            runner.retry_preparation(failed_at).is_err(),
13771            "old approvals cannot unlock the hold"
13772        );
13773        let mut task = crate::queue::Task::new(
13774            "retry".into(),
13775            "retry setup".into(),
13776            repo.clone(),
13777            crate::queue::Source::Human,
13778        );
13779        task.start(runner.state.id.clone());
13780        task.hold_machine(Some("setup failed".into()));
13781        task.release_operator("cli", None);
13782        runner
13783            .retry_preparation(task.resumption.as_ref().unwrap().at)
13784            .unwrap();
13785        runner.prep().await.unwrap();
13786        assert!(runner.state.prep_failure.is_none());
13787        assert_eq!(runner.state.prep_complete, Some(true));
13788        assert_eq!(runner.state.candidates.len(), 2);
13789        assert!(runner.state.worktree_root().join("judge-1").exists());
13790        for candidate in &runner.state.candidates {
13791            assert_eq!(
13792                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13793                git::tree_of(&repo, &seed).await.unwrap()
13794            );
13795        }
13796    }
13797
13798    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13799        let mut heads = Vec::new();
13800        for candidate in candidates {
13801            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13802        }
13803        heads
13804    }
13805
13806    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13807        ask_test_home();
13808        let tmp = tempfile::tempdir().unwrap();
13809        let repo = tmp.path().join("repo");
13810        std::fs::create_dir(&repo).unwrap();
13811        init_repo(&repo);
13812        let mut runner = runner_at(RunStatus::Reviewing);
13813        runner.state.repo = repo.clone();
13814        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13815        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13816        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13817            let branch = format!("candidate-{label}");
13818            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13819            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13820                .await
13821                .unwrap();
13822            if label == 'B' {
13823                std::fs::write(worktree.join("change"), "winner").unwrap();
13824                git::rescue_commit(&worktree, "candidate change")
13825                    .await
13826                    .unwrap();
13827            }
13828            runner.state.candidates.push(Candidate {
13829                index,
13830                label,
13831                agent: "alpha".into(),
13832                branch,
13833                worktree,
13834                summary: "completed reply".into(),
13835                stat: String::new(),
13836                files: usize::from(label == 'B'),
13837                commits: usize::from(label == 'B'),
13838                empty: label == 'A',
13839                failed: None,
13840                verified_noop: None,
13841                duration_ms: 1,
13842                folded: false,
13843            });
13844        }
13845        runner.state.judge_skipped = true;
13846        runner.state.tally = Some(Tally {
13847            first_choice: BTreeMap::new(),
13848            borda: BTreeMap::new(),
13849            winner: 'B',
13850            rankings: 0,
13851            unanimous_initial: false,
13852            deliberated: false,
13853            changed_votes: 0,
13854            unanimous_final: false,
13855            tie_break: None,
13856            judges: 0,
13857            present: 0,
13858            quorum: 0,
13859            met_quorum: true,
13860            uncontested: Some("only B produced a change".into()),
13861        });
13862        runner.state.save().unwrap();
13863        (tmp, runner)
13864    }
13865
13866    #[tokio::test]
13867    #[cfg(unix)]
13868    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13869        for (solo, failed_checkpoint) in
13870            [(true, false), (false, false), (true, true), (false, true)]
13871        {
13872            let (_tmp, mut runner) = preservation_resume_fixture().await;
13873            if solo {
13874                runner.state.candidates.retain(|c| c.label == 'A');
13875            }
13876            runner.state.candidates[0].empty = false;
13877            runner.state.candidates[0].summary.clear();
13878            runner.state.tally = None;
13879            runner.state.judge_skipped = false;
13880            runner.state.status = RunStatus::Implementing;
13881            runner.state.config.disk.min_free_bytes = u64::MAX;
13882            let mut implementer = spec("alpha");
13883            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13884            runner.roles.implementers = vec![implementer];
13885            let patch = runner.state.dir().join("artifacts/cand-A.patch");
13886            if failed_checkpoint {
13887                std::fs::create_dir_all(&patch).unwrap();
13888            }
13889            runner.state.save().unwrap();
13890            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13891            runner.state = RunState::load(&runner.state.id).unwrap();
13892            assert_eq!(runner.state.status, RunStatus::Blocked);
13893            assert!(runner.state.interrupted_implementations.contains(&'A'));
13894            assert_eq!(runner.state.candidates[0].commits, 0);
13895            assert!(!runner.state.candidates[0].empty);
13896            assert!(runner.state.candidates[0].failed.is_none());
13897            if failed_checkpoint {
13898                assert!(runner.state.candidate_preservation.contains_key(&'A'));
13899                std::fs::remove_dir(patch).unwrap();
13900            }
13901            runner.state.resource_block = None;
13902            runner.state.config.disk.min_free_bytes = 0;
13903            runner.roles.implementers[0].command = vec![
13904                "sh".into(),
13905                "-c".into(),
13906                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13907            ];
13908            runner.implement().await.unwrap();
13909            assert!(runner.state.interrupted_implementations.is_empty());
13910            assert!(runner.state.candidate_preservation.is_empty());
13911            assert_eq!(runner.state.candidates[0].commits, 1);
13912            assert!(!runner.state.candidates[0].empty);
13913            assert!(
13914                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13915                    .unwrap()
13916                    .contains("retried empty interrupted seat")
13917            );
13918            if !solo {
13919                assert_eq!(runner.state.candidates[1].commits, 1);
13920            }
13921        }
13922    }
13923
13924    #[tokio::test]
13925    #[cfg(unix)]
13926    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13927        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13928            let (_tmp, mut runner) = preservation_resume_fixture().await;
13929            runner.state.candidates.retain(|c| c.label == 'A');
13930            runner.state.candidates[0].empty = false;
13931            runner.state.candidates[0].summary.clear();
13932            runner.state.tally = None;
13933            runner.state.judge_skipped = false;
13934            runner.state.status = RunStatus::Implementing;
13935            // Force the real post-spawn watchdog, independently of available
13936            // disk space. Dirty/committed work predates its first measurement.
13937            runner.state.config.disk.min_free_bytes = u64::MAX;
13938            let wt = runner.state.candidates[0].worktree.clone();
13939            std::fs::write(
13940                wt.join("interrupted-source"),
13941                "retain implementation work\n",
13942            )
13943            .unwrap();
13944            if committed {
13945                git::rescue_commit(&wt, "implementation checkpoint")
13946                    .await
13947                    .unwrap();
13948            }
13949            let lock = git::git(
13950                &wt,
13951                &[
13952                    "rev-parse",
13953                    "--path-format=absolute",
13954                    "--git-path",
13955                    "index.lock",
13956                ],
13957            )
13958            .await
13959            .unwrap();
13960            if block_rescue {
13961                std::fs::write(&lock, "fixture-owned lock").unwrap();
13962            }
13963            let mut implementer = spec("alpha");
13964            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13965            runner.roles.implementers = vec![implementer];
13966            runner.state.save().unwrap();
13967            let result = runner.implement().await;
13968            assert_eq!(result.is_err(), block_rescue);
13969            assert_eq!(
13970                runner.state.resource_block.as_ref().unwrap().kind,
13971                "disk pressure"
13972            );
13973            assert!(
13974                runner
13975                    .state
13976                    .events
13977                    .iter()
13978                    .any(|e| e.message.contains("agent execution interrupted"))
13979            );
13980            assert!(
13981                runner
13982                    .state
13983                    .dir()
13984                    .join("artifacts/candidate-A-recovery/staged.patch")
13985                    .exists()
13986            );
13987            if block_rescue {
13988                let saved = RunState::load(&runner.state.id).unwrap();
13989                assert!(saved.candidate_preservation.contains_key(&'A'));
13990                assert!(!saved.candidates[0].empty);
13991                assert_eq!(
13992                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
13993                    "retain implementation work\n"
13994                );
13995                std::fs::remove_file(lock).unwrap();
13996                runner.state = saved;
13997            } else {
13998                assert_eq!(runner.state.status, RunStatus::Blocked);
13999                assert_eq!(runner.state.candidates[0].commits, 1);
14000            }
14001            // Recovery must save the candidate before deciding whether another
14002            // implementation is needed. No implementer is available on retry.
14003            runner.roles.implementers.clear();
14004            runner.state.config.disk.min_free_bytes = 0;
14005            runner.state.resource_block = None;
14006            runner.implement().await.unwrap();
14007            assert!(runner.state.candidate_preservation.is_empty());
14008            assert_eq!(runner.state.candidates[0].commits, 1);
14009            assert!(!runner.state.candidates[0].empty);
14010            assert!(git::is_clean(&wt).await.unwrap());
14011            assert!(
14012                crate::run::read_artifact(&runner.state, "cand-A.patch")
14013                    .unwrap()
14014                    .contains("retain implementation work")
14015            );
14016        }
14017    }
14018
14019    #[tokio::test]
14020    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14021        let (_tmp, mut runner) = preservation_resume_fixture().await;
14022        // Schema 15 already confirmed the unfolded no-change result.
14023        runner.implement().await.unwrap();
14024        assert!(runner.state.candidate_preservation.is_empty());
14025        assert!(
14026            !runner
14027                .state
14028                .dir()
14029                .join("artifacts/candidate-A-recovery")
14030                .exists()
14031        );
14032        runner.fold_losers().await.unwrap();
14033        assert!(!runner.state.candidates[0].worktree.exists());
14034        for schema in [14, crate::run::SCHEMA] {
14035            runner.state.schema = schema;
14036            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14037            runner.state.save().unwrap();
14038            runner.state = RunState::load(&runner.state.id).unwrap();
14039            runner.implement().await.unwrap();
14040            assert!(runner.state.candidate_preservation.is_empty());
14041            assert!(runner.state.candidates[0].folded);
14042            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14043        }
14044    }
14045
14046    #[tokio::test]
14047    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14048        let (_tmp, mut runner) = preservation_resume_fixture().await;
14049        let wt = runner.state.candidates[0].worktree.clone();
14050        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14051        runner.state.schema = 14;
14052        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14053        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14054        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14055        for _ in 0..2 {
14056            assert!(runner.implement().await.is_err());
14057            assert_eq!(runner.state.status, RunStatus::Blocked);
14058            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14059            assert!(
14060                runner.state.candidates[0].empty,
14061                "old verdict must remain unusable"
14062            );
14063            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14064            assert!(runner.state.judge_skipped);
14065            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14066            assert!(
14067                crate::run::read_artifact(&runner.state, "cand-A.patch")
14068                    .unwrap()
14069                    .contains("owned edits")
14070            );
14071            assert!(wt.exists());
14072            assert_eq!(
14073                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14074                    .await
14075                    .unwrap(),
14076                1
14077            );
14078            runner.state = RunState::load(&runner.state.id).unwrap();
14079        }
14080    }
14081
14082    #[tokio::test]
14083    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14084        let (_tmp, mut runner) = preservation_resume_fixture().await;
14085        let wt = runner.state.candidates[0].worktree.clone();
14086        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14087        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14088        let index_lock = PathBuf::from(
14089            git::git(
14090                &wt,
14091                &[
14092                    "rev-parse",
14093                    "--path-format=absolute",
14094                    "--git-path",
14095                    "index.lock",
14096                ],
14097            )
14098            .await
14099            .unwrap(),
14100        );
14101        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14102        runner.state.schema = 14;
14103        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14104        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14105        assert!(runner.implement().await.is_err());
14106        assert!(runner.after_implement().is_err());
14107        assert!(runner.fold_losers().await.is_err());
14108        assert_eq!(
14109            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14110            "failed seat edits"
14111        );
14112        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14113        runner.state = RunState::load(&runner.state.id).unwrap();
14114        std::fs::remove_file(index_lock).unwrap();
14115        // Recovery finds changes that the old uncontested result never judged.
14116        assert!(runner.implement().await.is_err());
14117        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14118        assert!(runner.state.candidates[0].empty);
14119        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14120        assert!(
14121            crate::run::read_artifact(&runner.state, "cand-A.patch")
14122                .unwrap()
14123                .contains("failed seat edits")
14124        );
14125        assert!(wt.exists());
14126    }
14127
14128    #[tokio::test]
14129    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14130        let (_tmp, mut runner) = preservation_resume_fixture().await;
14131        let repo = runner.state.repo.clone();
14132        let common = git::git(
14133            &repo,
14134            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14135        )
14136        .await
14137        .unwrap();
14138        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14139        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14140        assert!(runner.fold_losers().await.is_err());
14141        assert!(!runner.state.candidates[0].worktree.exists());
14142        assert!(!runner.state.candidates[0].folded);
14143        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14144        runner.state = RunState::load(&runner.state.id).unwrap();
14145        std::fs::remove_file(branch_lock).unwrap();
14146        let judge = runner.state.worktree_root().join("judge-1");
14147        git::worktree_add_detached(&repo, &judge, "main")
14148            .await
14149            .unwrap();
14150        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14151        runner.roles.judges = vec![spec("judge")];
14152        assert!(runner.fold_losers().await.is_err());
14153        runner.state = RunState::load(&runner.state.id).unwrap();
14154        assert!(
14155            runner.state.candidates[0].folded,
14156            "candidate progress saved before judge failure"
14157        );
14158        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14159        assert!(judge.join("uncommitted").exists());
14160        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14161        runner.fold_losers().await.unwrap();
14162        assert!(!judge.exists());
14163        runner.fold_losers().await.unwrap();
14164    }
14165
14166    #[tokio::test]
14167    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14168        let (_tmp, mut runner) = preservation_resume_fixture().await;
14169        runner.state.candidates.retain(|c| c.label == 'B');
14170        runner.state.tally = None;
14171        runner.state.judge_skipped = false;
14172        runner.state.prep_complete = Some(true);
14173        runner.state.config.graph.advise = false;
14174        runner.state.config.graph.selected_implementer = Some("alpha".into());
14175        let wt = runner.state.candidates[0].worktree.clone();
14176        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14177        runner
14178            .state
14179            .candidate_preservation
14180            .insert('B', "awaiting persistence".into());
14181        let index_lock = git::git(
14182            &wt,
14183            &[
14184                "rev-parse",
14185                "--path-format=absolute",
14186                "--git-path",
14187                "index.lock",
14188            ],
14189        )
14190        .await
14191        .unwrap();
14192        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14193        runner.state.save().unwrap();
14194        assert!(runner.execute_graph().await.is_err());
14195        assert_eq!(runner.state.status, RunStatus::Blocked);
14196        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14197        std::fs::remove_file(index_lock).unwrap();
14198        // Park at the next graph boundary: this tests the real selected-seat
14199        // hold guard without dispatching reviewers or publishing anything.
14200        for _ in 0..2 {
14201            runner.state = RunState::load(&runner.state.id).unwrap();
14202            runner.pause.park();
14203            runner.execute_graph().await.unwrap();
14204            assert_eq!(runner.state.status, RunStatus::Implementing);
14205            assert!(runner.state.candidate_preservation.is_empty());
14206            assert_eq!(runner.state.candidates[0].commits, 2);
14207            assert!(runner.state.tally.is_none());
14208        }
14209        runner.implement().await.unwrap();
14210        assert_eq!(runner.state.status, RunStatus::Judging);
14211        // A genuine selected-implementer hold must still require release.
14212        runner.state.status = RunStatus::Blocked;
14213        runner.state.candidates[0].failed = Some("implementation failed".into());
14214        runner.state.save().unwrap();
14215        let error = runner.execute_graph().await.unwrap_err();
14216        assert!(
14217            error
14218                .to_string()
14219                .contains("selected implementation is held")
14220        );
14221    }
14222
14223    #[tokio::test]
14224    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14225        let (_tmp, mut runner) = preservation_resume_fixture().await;
14226        let repo = runner.state.repo.clone();
14227        let candidate = runner.state.candidates[0].clone();
14228        // Remove only the disposable fixture directory, leaving Git's linked
14229        // worktree registration and branch intact, as in a vanished checkout.
14230        let registered = git::worktree_holding(&repo, &candidate.branch)
14231            .await
14232            .unwrap()
14233            .unwrap();
14234        assert_eq!(
14235            registered.canonicalize().unwrap(),
14236            candidate.worktree.canonicalize().unwrap()
14237        );
14238        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14239        assert_eq!(
14240            git::worktree_holding(&repo, &candidate.branch)
14241                .await
14242                .unwrap(),
14243            Some(registered)
14244        );
14245        assert!(
14246            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14247            "the stale registration initially keeps the branch checked out"
14248        );
14249        runner.fold_losers().await.unwrap();
14250        assert!(runner.state.candidates[0].folded);
14251        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14252        assert_eq!(
14253            git::worktree_holding(&repo, &candidate.branch)
14254                .await
14255                .unwrap(),
14256            None
14257        );
14258        runner.state = RunState::load(&runner.state.id).unwrap();
14259        runner.fold_losers().await.unwrap();
14260    }
14261
14262    #[tokio::test]
14263    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14264        let (_tmp, mut runner) = preservation_resume_fixture().await;
14265        let repo = runner.state.repo.clone();
14266        let candidate = runner.state.candidates[0].clone();
14267        let winner = runner.state.candidates[1].clone();
14268        let registered = git::worktree_holding(&repo, &candidate.branch)
14269            .await
14270            .unwrap()
14271            .unwrap();
14272        assert_eq!(
14273            registered.canonicalize().unwrap(),
14274            candidate.worktree.canonicalize().unwrap()
14275        );
14276        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14277        assert_eq!(
14278            git::worktree_holding(&repo, &candidate.branch)
14279                .await
14280                .unwrap(),
14281            Some(registered)
14282        );
14283        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14284
14285        for _ in 0..2 {
14286            fold_run(&mut runner.state, false, &crate::run::home())
14287                .await
14288                .unwrap();
14289            runner.state = RunState::load(&runner.state.id).unwrap();
14290            assert!(runner.state.candidates[0].folded);
14291            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14292            assert_eq!(
14293                git::worktree_holding(&repo, &candidate.branch)
14294                    .await
14295                    .unwrap(),
14296                None
14297            );
14298            assert!(!runner.state.candidates[1].folded);
14299            assert!(winner.worktree.exists());
14300            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14301        }
14302    }
14303
14304    #[tokio::test]
14305    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14306        ask_test_home();
14307        let tmp = tempfile::tempdir().unwrap();
14308        let repo = tmp.path().join("repo");
14309        std::fs::create_dir(&repo).unwrap();
14310        init_repo(&repo);
14311        let mut runner = runner_at(RunStatus::Implementing);
14312        runner.state.repo = repo.clone();
14313        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14314        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14315        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14316            let branch = format!("candidate-{label}");
14317            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14318            git::worktree_add_branch(&repo, &wt, &branch, "main")
14319                .await
14320                .unwrap();
14321            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14322            git::git(&wt, &["add", "README.md"]).await.unwrap();
14323            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14324            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14325            runner.state.candidates.push(Candidate {
14326                index,
14327                label,
14328                agent: "alpha".to_owned(),
14329                branch,
14330                worktree: wt,
14331                summary: "completed implementation".to_owned(),
14332                stat: String::new(),
14333                files: 0,
14334                commits: 0,
14335                empty: false,
14336                failed: None,
14337                verified_noop: None,
14338                duration_ms: 42,
14339                folded: false,
14340            });
14341            runner
14342                .state
14343                .candidate_preservation
14344                .insert(label, "awaiting persistence".to_owned());
14345        }
14346        let wt = runner.state.candidates[0].worktree.clone();
14347        let index_lock = git::git(
14348            &wt,
14349            &[
14350                "rev-parse",
14351                "--path-format=absolute",
14352                "--git-path",
14353                "index.lock",
14354            ],
14355        )
14356        .await
14357        .unwrap();
14358        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14359        runner.state.save().unwrap();
14360        assert!(runner.persist_candidates().await.is_err());
14361        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14362        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14363        assert_eq!(
14364            runner.state.candidates[1].commits, 1,
14365            "other results are still collected"
14366        );
14367        assert!(
14368            !runner.state.candidates[0].empty,
14369            "failed rescue is not no change"
14370        );
14371        assert_eq!(
14372            runner.state.candidates[0].summary,
14373            "completed implementation"
14374        );
14375        assert!(runner.after_implement().is_err());
14376        let home = tmp.path().join("fold-home");
14377        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14378        assert!(wt.join("new-file").exists());
14379        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14380        assert!(recovery.join("staged.patch").exists());
14381        assert!(recovery.join("unstaged.patch").exists());
14382        assert_eq!(
14383            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14384            "untracked edits\n"
14385        );
14386        runner.state.begin_pass();
14387        runner.state.end_pass_parked(ParkCause::Upgrade {
14388            from: "1.0".into(),
14389            to: Some("1.1".into()),
14390        });
14391        runner.state.save().unwrap();
14392        runner.state = RunState::load(&runner.state.id).unwrap();
14393        assert!(matches!(
14394            runner.state.passes.last().unwrap().end,
14395            Some(crate::run::PassEnd::Parked {
14396                cause: ParkCause::Upgrade { .. }
14397            })
14398        ));
14399        std::fs::remove_file(index_lock).unwrap();
14400        // No implementer is configured: a persistence retry cannot dispatch a seat.
14401        runner.implement().await.unwrap();
14402        assert!(runner.state.candidate_preservation.is_empty());
14403        assert!(
14404            runner
14405                .state
14406                .candidates
14407                .iter()
14408                .all(|c| c.commits == 1 && !c.empty)
14409        );
14410        assert!(git::is_clean(&wt).await.unwrap());
14411        assert!(
14412            crate::run::read_artifact(&runner.state, "cand-A.patch")
14413                .unwrap()
14414                .contains("later tracked edits")
14415        );
14416    }
14417
14418    #[tokio::test]
14419    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14420        ask_test_home();
14421        let tmp = tempfile::tempdir().unwrap();
14422        let repo = tmp.path().join("repo");
14423        std::fs::create_dir(&repo).unwrap();
14424        init_repo(&repo);
14425        let wt = tmp.path().join("candidate");
14426        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14427            .await
14428            .unwrap();
14429        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14430        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14431        assert!(wt.join("uncommitted").exists());
14432        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14433        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14434        let mut runner = runner_at(RunStatus::Implementing);
14435        runner.state.repo = repo.clone();
14436        runner.state.base_commit = "missing-base".to_owned();
14437        runner.state.candidates.push(Candidate {
14438            index: 0,
14439            label: 'A',
14440            agent: "alpha".into(),
14441            branch: "candidate".into(),
14442            worktree: wt.clone(),
14443            summary: "completed".into(),
14444            stat: String::new(),
14445            files: 0,
14446            commits: 0,
14447            empty: false,
14448            failed: None,
14449            verified_noop: None,
14450            duration_ms: 1,
14451            folded: false,
14452        });
14453        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14454        assert!(
14455            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14456                .await
14457                .is_err()
14458        );
14459        assert!(!runner.state.candidates[0].folded);
14460        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14461        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14462        runner
14463            .state
14464            .candidate_preservation
14465            .insert('A', "pending".into());
14466        assert!(runner.persist_candidates().await.is_err());
14467        assert!(!runner.state.candidates[0].empty);
14468        assert!(!runner.state.candidates[0].folded);
14469        assert!(wt.exists());
14470        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14471        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14472        std::fs::create_dir_all(&patch_path).unwrap();
14473        assert!(
14474            runner.persist_candidates().await.is_err(),
14475            "artifact failure stays pending"
14476        );
14477        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14478        assert!(!runner.state.candidates[0].empty);
14479        std::fs::remove_dir(patch_path).unwrap();
14480        runner.persist_candidates().await.unwrap();
14481        assert!(
14482            runner.state.candidates[0].empty,
14483            "only successful inspection confirms no change"
14484        );
14485    }
14486
14487    fn init_repo(dir: &Path) {
14488        let run = |args: &[&str]| {
14489            let out = std::process::Command::new("git")
14490                .args(args)
14491                .current_dir(dir)
14492                .quiet()
14493                .output()
14494                .expect("spawn git");
14495            assert!(
14496                out.status.success(),
14497                "git {args:?} failed: {}",
14498                String::from_utf8_lossy(&out.stderr)
14499            );
14500        };
14501        run(&["init", "-b", "main"]);
14502        run(&["config", "user.name", "magi test"]);
14503        run(&["config", "user.email", "magi@example.com"]);
14504        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14505        run(&["add", "-A"]);
14506        run(&["commit", "-m", "init"]);
14507    }
14508
14509    // `settle_questions` is what closes the ghost the phone showed: a run's
14510    // seat asked something, the run then ended, and nothing was left to
14511    // abandon the question it left `open`. `HOME` is a process-wide
14512    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14513    // first time it runs in the binary — every test below still reaches the
14514    // same directory whichever call won, and each gets its own run id from
14515    // `RunState::new`, so they never collide there.
14516    fn ask_test_home() {
14517        crate::run::pin_test_home();
14518    }
14519
14520    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14521    /// reads nothing else off it.
14522    fn runner_at(status: RunStatus) -> Runner {
14523        let mut state = RunState::new(
14524            PathBuf::from("/nonexistent/repo"),
14525            "main".to_owned(),
14526            "deadbeef".to_owned(),
14527            "task".to_owned(),
14528            Config::default(),
14529        );
14530        // These fixtures share one home and save concurrently. Short clock
14531        // IDs can collide, making an ownership test read another test's run.
14532        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14533        state.id = format!(
14534            "20000101-000000-{:04x}",
14535            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14536        );
14537        state.status = status;
14538        Runner {
14539            state,
14540            roles: ResolvedRoles {
14541                implementers: Vec::new(),
14542                judges: Vec::new(),
14543                reviewers: Vec::new(),
14544                fixer: None,
14545                conductor: conductor(),
14546                implementer_roster: Vec::new(),
14547                judge_roster: Vec::new(),
14548                reviewer_roster: Vec::new(),
14549            },
14550            sem: Arc::new(Semaphore::new(1)),
14551            pause: Pause::new(),
14552            interrupt: Pause::new(),
14553        }
14554    }
14555
14556    #[test]
14557    fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14558        crate::run::pin_test_home();
14559        let mut runner = runner_at(RunStatus::Gating);
14560        runner.state.save().unwrap();
14561        // run.json can commit, but the subsequent lease metadata write fails.
14562        // Cleanup must run after releasing the short ownership lock.
14563        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14564        assert!(runner.claim_driver().is_err());
14565        assert!(runner.state.driver_exited);
14566        assert_eq!(runner.state.driver_pid, None);
14567        let disk = RunState::load(&runner.state.id).unwrap();
14568        assert!(disk.driver_exited);
14569        assert_eq!(disk.driver_pid, None);
14570        assert_eq!(disk.status, RunStatus::Gating);
14571    }
14572
14573    #[tokio::test]
14574    async fn an_early_graph_error_releases_its_driver() {
14575        crate::run::pin_test_home();
14576        let mut runner = runner_at(RunStatus::Blocked);
14577        runner.state.config.graph.selected_implementer = Some("selected".into());
14578        runner.state.save().unwrap();
14579        assert!(runner.execute().await.is_err());
14580        let disk = RunState::load(&runner.state.id).unwrap();
14581        assert!(disk.driver_exited);
14582        assert_eq!(disk.driver_pid, None);
14583        assert_eq!(disk.status, RunStatus::Blocked);
14584    }
14585
14586    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14587        crate::run::MergeRepair {
14588            question: "repair-question".into(),
14589            head: "old-head".into(),
14590            pr: "https://github.com/o/r/pull/42".into(),
14591            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14592            gate: vec![outcome(Some(0), false)],
14593            gate_head: Some("old-head".into()),
14594            gate_fixes: Vec::new(),
14595            pre_gate: Vec::new(),
14596            phase,
14597            failure: None,
14598        }
14599    }
14600
14601    #[tokio::test]
14602    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14603        crate::run::pin_test_home();
14604        let mut runner = runner_at(RunStatus::Landing);
14605        runner.state.merge_repairs.push(merge_repair_request(
14606            crate::run::MergeRepairPhase::Requested,
14607        ));
14608        runner.state.save().unwrap();
14609        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14610        assert!(
14611            runner
14612                .execute()
14613                .await
14614                .unwrap_err()
14615                .to_string()
14616                .contains("already running")
14617        );
14618        assert_eq!(
14619            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14620            crate::run::MergeRepairPhase::Requested
14621        );
14622        drop(claim);
14623        let mut disk = runner.state.clone();
14624        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14625        disk.save().unwrap();
14626        assert!(
14627            runner
14628                .execute()
14629                .await
14630                .unwrap_err()
14631                .to_string()
14632                .contains("decision changed")
14633        );
14634        assert_eq!(
14635            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14636            crate::run::MergeRepairPhase::Complete
14637        );
14638    }
14639
14640    #[test]
14641    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14642        crate::run::pin_test_home();
14643        let mut runner = runner_at(RunStatus::Landing);
14644        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14645        runner.state.reviews = request.reviews.clone();
14646        runner.state.gate = request.gate.clone();
14647        runner.state.gate_ran = true;
14648        runner.state.land_approval = Some(crate::run::LandApproval {
14649            question: "old".into(),
14650            head: "old-head".into(),
14651        });
14652        runner.state.merge_repairs.push(request);
14653        runner.begin_merge_repair_review(0).unwrap();
14654        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14655        assert!(!runner.state.gate_status().ok());
14656        assert!(runner.state.land_approval.is_none());
14657        let disk = RunState::load(&runner.state.id).unwrap();
14658        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14659        assert!(disk.merge_repairs[0].gate[0].ok());
14660        assert_eq!(
14661            disk.merge_repairs[0].phase,
14662            crate::run::MergeRepairPhase::Reviewing
14663        );
14664        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14665        assert!(disk.reviews.is_empty());
14666    }
14667
14668    #[tokio::test]
14669    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14670        crate::run::pin_test_home();
14671        for phase in [
14672            crate::run::MergeRepairPhase::Fixing,
14673            crate::run::MergeRepairPhase::Requested,
14674        ] {
14675            let mut runner = runner_at(RunStatus::Landing);
14676            runner.state.config.graph.land_rounds = 0;
14677            runner.state.merge_repairs.push(merge_repair_request(phase));
14678            runner.state.save().unwrap();
14679            assert!(!runner.repair_merge_review().await.unwrap());
14680            assert_eq!(runner.state.status, RunStatus::Blocked);
14681            assert_eq!(
14682                runner.state.merge_repairs[0].phase,
14683                crate::run::MergeRepairPhase::Failed
14684            );
14685            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14686            assert!(
14687                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14688                    "interrupted"
14689                } else {
14690                    "budget exhausted"
14691                })
14692            );
14693            assert!(runner.state.merge.is_none());
14694        }
14695    }
14696
14697    /// `park_here` folding in the reason `Pause::park_because` recorded -
14698    /// this is what lets an operator reading a run's events tell an
14699    /// interrupt-driven park from an ordinary shutdown park.
14700    #[test]
14701    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14702        crate::run::pin_test_home();
14703        let mut runner = runner_at(RunStatus::Implementing);
14704        let interrupt = Pause::new();
14705        runner.watch_interrupt(interrupt.clone());
14706
14707        interrupt.park_because("task a1b2 asked to run first");
14708
14709        assert!(runner.park_here().expect("park_here"));
14710        assert!(runner.state.parked);
14711        let last = runner.state.events.last().expect("a park event");
14712        assert_eq!(last.node, "park");
14713        assert!(
14714            last.message.contains("task a1b2 asked to run first"),
14715            "expected the interrupt reason in {:?}",
14716            last.message
14717        );
14718    }
14719
14720    /// The cause is stored on the pass when it parks, and the first cause a
14721    /// shared pause was given is the one that stays.
14722    #[test]
14723    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14724        crate::run::pin_test_home();
14725        let mut runner = runner_at(RunStatus::Implementing);
14726        runner.state.begin_pass();
14727        let shutdown = Pause::new();
14728        runner.on_pause(shutdown.clone());
14729        let upgrade = ParkCause::Upgrade {
14730            from: "0.121.0".to_owned(),
14731            to: Some("0.121.1".to_owned()),
14732        };
14733        shutdown.park_for(upgrade.clone());
14734        shutdown.park_for(ParkCause::Operator);
14735
14736        assert!(runner.park_here().expect("park_here"));
14737        assert_eq!(
14738            runner.state.passes[0].end,
14739            Some(crate::run::PassEnd::Parked { cause: upgrade })
14740        );
14741        let last = runner.state.events.last().expect("a park event");
14742        assert!(last.message.contains("0.121.0"), "{}", last.message);
14743    }
14744
14745    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14746    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14747    /// cleared) must not make a *different* run - one only watching its own,
14748    /// unshared interrupt `Pause` - see itself as parked. If a future change
14749    /// ever collapsed these back into one handle, the interrupt scheduler
14750    /// would park every run for the rest of the daemon's life, not just the
14751    /// one it meant to interrupt.
14752    #[test]
14753    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14754        crate::run::pin_test_home();
14755        let mut runner = runner_at(RunStatus::Implementing);
14756        let shutdown = Pause::new();
14757        runner.on_pause(shutdown.clone());
14758        let interrupt = Pause::new();
14759        runner.watch_interrupt(interrupt.clone());
14760
14761        // Nobody has asked for anything yet.
14762        assert!(!runner.park_here().expect("park_here"));
14763        assert!(!runner.state.parked);
14764
14765        // Only the interrupt handle fires; the shutdown handle stays clear.
14766        interrupt.park_because("test");
14767        assert!(!shutdown.parked());
14768        assert!(runner.park_here().expect("park_here"));
14769    }
14770
14771    /// The property every prior attempt at this feature failed to pin down:
14772    /// asking a run to park while one of its nodes has a real, in-flight
14773    /// async operation running (an agent call, in production) must not cut
14774    /// that operation short. `park_here` is only ever consulted *between*
14775    /// `execute`'s node calls - see its own doc - so nothing inside a node
14776    /// can observe a park request until the node itself returns. This proves
14777    /// that structurally, with real `tokio` concurrency and a channel
14778    /// handshake (never a sleep, which would only prove "usually", not
14779    /// "cannot"): the "node" below reports that it has genuinely started,
14780    /// and only then is the park requested; the node still has to be told to
14781    /// finish before `park_here` is ever called, exactly mirroring every
14782    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14783    /// in `execute`.
14784    #[tokio::test]
14785    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14786        crate::run::pin_test_home();
14787        let mut runner = runner_at(RunStatus::Implementing);
14788        let interrupt = Pause::new();
14789        runner.watch_interrupt(interrupt.clone());
14790
14791        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14792        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14793
14794        // Stands in for one node's in-flight agent call: it proves it has
14795        // genuinely started, then blocks - exactly as a spawned CLI process
14796        // does - until told to finish.
14797        let node = async move {
14798            started_tx.send(()).expect("send started");
14799            finish_rx.await.expect("recv finish");
14800            "node finished"
14801        };
14802
14803        let interrupter = async move {
14804            started_rx.await.expect("recv started");
14805            // The call is now genuinely in flight. Ask it to park.
14806            interrupt.park_because("higher-priority task waiting");
14807            // Nothing the node does can observe this yet - there is no
14808            // check inside it, by construction - so let the executor run
14809            // anything pending and then let the node finish on its own.
14810            tokio::task::yield_now().await;
14811            finish_tx.send(()).expect("send finish");
14812        };
14813
14814        let (node_result, ()) = tokio::join!(node, interrupter);
14815        assert_eq!(
14816            node_result, "node finished",
14817            "the in-flight call ran to completion"
14818        );
14819
14820        // Only now, at the boundary the real `execute` would check right
14821        // after this node, does the park take effect.
14822        assert!(runner.park_here().expect("park_here"));
14823        assert!(runner.state.parked);
14824    }
14825
14826    /// A run parked mid-competition carries every field it had accumulated
14827    /// through the exact same disk round-trip an ordinary resume uses -
14828    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
14829    /// Nothing about parking for an interrupt is a special case of that path;
14830    /// this is what proves it rather than assuming it.
14831    #[test]
14832    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14833        crate::run::pin_test_home();
14834        let mut runner = runner_at(RunStatus::Judging);
14835        // `Runner::resume` re-resolves roles from the saved config, which
14836        // refuses an empty roster - give it the same minimal one `conductor`
14837        // itself uses.
14838        runner.state.config.agents = vec![conductor()];
14839        runner.state.candidates = vec![Candidate {
14840            index: 0,
14841            label: 'A',
14842            agent: "alpha".to_owned(),
14843            branch: "magi/x/A".to_owned(),
14844            worktree: PathBuf::from("/nonexistent/worktree"),
14845            summary: "did the thing".to_owned(),
14846            stat: "1 file changed".to_owned(),
14847            files: 1,
14848            commits: 1,
14849            empty: false,
14850            failed: None,
14851            verified_noop: None,
14852            duration_ms: 1234,
14853            folded: false,
14854        }];
14855        let run_id = runner.state.id.clone();
14856
14857        let interrupt = Pause::new();
14858        runner.watch_interrupt(interrupt.clone());
14859        interrupt.park_because("task c3d4 asked to run first");
14860        assert!(runner.park_here().expect("park_here"));
14861
14862        let resumed = Runner::resume(&run_id).expect("resume");
14863        assert_eq!(resumed.state.candidates.len(), 1);
14864        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14865        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14866        assert_eq!(resumed.state.status, runner.state.status);
14867        assert!(
14868            resumed.state.parked,
14869            "still parked until `execute` actually walks the graph again"
14870        );
14871        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14872    }
14873
14874    /// A fresh open question on `run`, stored and handed back for assertions.
14875    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14876        let mut q = ask::Question::new(
14877            run.to_owned(),
14878            "implement".to_owned(),
14879            "impl-A".to_owned(),
14880            "Which storage backend should the cache use?".to_owned(),
14881            String::new(),
14882            vec!["SQLite".to_owned(), "Redis".to_owned()],
14883        );
14884        store.put(&mut q).unwrap();
14885        q
14886    }
14887
14888    #[test]
14889    fn a_failed_runs_open_question_is_abandoned() {
14890        ask_test_home();
14891        let store = ask::Questions::open();
14892        let mut runner = runner_at(RunStatus::Failed);
14893        let run = runner.state.id.clone();
14894        let q = ask_open_question(&store, &run);
14895
14896        runner.settle_questions();
14897
14898        let back = store.get(&q.id).unwrap();
14899        assert!(
14900            !back.status.open(),
14901            "the seat that asked died with the run; nobody is left to read an answer"
14902        );
14903        assert!(
14904            back.detail.contains(&run) && back.detail.contains("failed"),
14905            "the reason names what the run became, not just that it is gone: {}",
14906            back.detail
14907        );
14908    }
14909
14910    #[test]
14911    fn a_merged_runs_open_question_is_abandoned_too() {
14912        ask_test_home();
14913        let store = ask::Questions::open();
14914        // A run that finishes cleanly still leaves nobody to read an answer -
14915        // this is not only a failure-path cleanup.
14916        for status in [RunStatus::Merged, RunStatus::Ready] {
14917            let mut runner = runner_at(status);
14918            let run = runner.state.id.clone();
14919            let q = ask_open_question(&store, &run);
14920
14921            runner.settle_questions();
14922
14923            let back = store.get(&q.id).unwrap();
14924            assert!(
14925                !back.status.open(),
14926                "{status:?} run's question must not outlive the run"
14927            );
14928        }
14929    }
14930
14931    #[test]
14932    fn a_still_resumable_runs_open_question_is_left_alone() {
14933        ask_test_home();
14934        let store = ask::Questions::open();
14935        // `Blocked` and `Stalled` can still be resumed — the candidates, the
14936        // review round and the seat sessions are all still on disk — so a
14937        // question asked mid-round may yet get a real answer from a real
14938        // resume. Sweeping it here would be exactly the failure mode this
14939        // whole feature exists to avoid on the other side.
14940        for status in [RunStatus::Blocked, RunStatus::Stalled] {
14941            let mut runner = runner_at(status);
14942            let run = runner.state.id.clone();
14943            let q = ask_open_question(&store, &run);
14944
14945            runner.settle_questions();
14946
14947            let back = store.get(&q.id).unwrap();
14948            assert!(
14949                back.status.open(),
14950                "{status:?} is still alive; the question must still be waiting"
14951            );
14952        }
14953    }
14954
14955    #[test]
14956    fn settle_questions_never_touches_an_already_answered_question() {
14957        ask_test_home();
14958        let store = ask::Questions::open();
14959        let mut runner = runner_at(RunStatus::Failed);
14960        let run = runner.state.id.clone();
14961        let mut q = ask_open_question(&store, &run);
14962        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14963            .unwrap();
14964        store.put(&mut q).unwrap();
14965
14966        // Called twice, the way a crash-recovered daemon reclaim and the
14967        // graph's own cleanup both can for the same run — `abandon_for_run`
14968        // only ever touches what is still open, so this must be inert both
14969        // times, not merely the second.
14970        runner.settle_questions();
14971        runner.settle_questions();
14972
14973        let back = store.get(&q.id).unwrap();
14974        assert_eq!(
14975            back.status,
14976            ask::QuestionStatus::Answered,
14977            "a real answer is a decision on record, never overwritten by a sweep"
14978        );
14979    }
14980
14981    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
14982    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
14983    /// without merging, whose winner is still the operator's answer to read.
14984    /// Nothing previously called `fold_run` itself with a real `tally`, so
14985    /// this is the first test to pin down the one distinction the whole
14986    /// automatic-fold feature depends on: the winner's worktree and branch
14987    /// must survive, everything else sharing the run's worktree bay - a
14988    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
14989    /// second sweep treats every non-winner directory under the bay alike -
14990    /// must not.
14991    #[tokio::test]
14992    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14993        crate::run::pin_test_home();
14994        let tmp = tempfile::tempdir().expect("tempdir");
14995        let repo = tmp.path().join("repo");
14996        std::fs::create_dir_all(&repo).unwrap();
14997        init_repo(&repo);
14998
14999        let mut config = Config::default();
15000        config.graph.worktree_root = Some(tmp.path().join("wt"));
15001
15002        let mut state = RunState::new(
15003            repo.clone(),
15004            "main".to_owned(),
15005            "deadbeef".to_owned(),
15006            "task".to_owned(),
15007            config,
15008        );
15009        let root = state.worktree_root();
15010        let wt_a = root.join("cand-A");
15011        let wt_b = root.join("cand-B");
15012        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15013            .await
15014            .expect("worktree A");
15015        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15016            .await
15017            .expect("worktree B");
15018
15019        state.candidates = vec![
15020            Candidate {
15021                index: 0,
15022                label: 'A',
15023                agent: "alpha".to_owned(),
15024                branch: "magi/x/A".to_owned(),
15025                worktree: wt_a.clone(),
15026                summary: String::new(),
15027                stat: String::new(),
15028                files: 0,
15029                commits: 0,
15030                empty: false,
15031                failed: None,
15032                verified_noop: None,
15033                duration_ms: 0,
15034                folded: false,
15035            },
15036            Candidate {
15037                index: 1,
15038                label: 'B',
15039                agent: "beta".to_owned(),
15040                branch: "magi/x/B".to_owned(),
15041                worktree: wt_b.clone(),
15042                summary: String::new(),
15043                stat: String::new(),
15044                files: 0,
15045                commits: 0,
15046                empty: false,
15047                failed: None,
15048                verified_noop: None,
15049                duration_ms: 0,
15050                folded: false,
15051            },
15052        ];
15053        state.tally = Some(Tally {
15054            first_choice: BTreeMap::from([('A', 1)]),
15055            borda: BTreeMap::new(),
15056            winner: 'A',
15057            rankings: 1,
15058            unanimous_initial: true,
15059            deliberated: false,
15060            changed_votes: 0,
15061            unanimous_final: true,
15062            tie_break: None,
15063            judges: 1,
15064            present: 1,
15065            quorum: 1,
15066            met_quorum: true,
15067            uncontested: None,
15068        });
15069        state.status = RunStatus::Ready;
15070
15071        fold_run(&mut state, false, &crate::run::home())
15072            .await
15073            .expect("fold_run");
15074
15075        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15076        assert!(
15077            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15078            "the unmerged winner's branch survives"
15079        );
15080        assert!(
15081            !state.candidates[0].folded,
15082            "the winner is not marked folded"
15083        );
15084
15085        assert!(!wt_b.exists(), "the loser's worktree is removed");
15086        assert!(
15087            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15088            "the loser's branch is removed"
15089        );
15090        assert!(state.candidates[1].folded, "the loser is marked folded");
15091    }
15092
15093    #[tokio::test]
15094    #[cfg(unix)]
15095    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15096        use std::os::fd::AsRawFd as _;
15097        use std::time::Duration;
15098
15099        let tmp = tempfile::tempdir().unwrap();
15100        let repo = tmp.path().join("repo");
15101        std::fs::create_dir_all(&repo).unwrap();
15102        init_repo(&repo);
15103        let common = repo.join(".git");
15104        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15105        let home = tmp.path().join("home");
15106        let mut config = Config::default();
15107        config.coordination = crate::config::test_coordination();
15108        config.graph.worktree_root = Some(tmp.path().join("wt"));
15109        config.graph.timeout_resource_wait = 1;
15110        let mut state = RunState::new(
15111            repo.clone(),
15112            "main".into(),
15113            "deadbeef".into(),
15114            "task".into(),
15115            config,
15116        );
15117        let root = state.worktree_root();
15118        let candidate = root.join("cand-A");
15119        let reviewer = root.join("review-1");
15120        let branch = state.branch_for('A');
15121        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15122            .await
15123            .unwrap();
15124        git::worktree_add_detached(&repo, &reviewer, "main")
15125            .await
15126            .unwrap();
15127        git::acquire_worktree_config(&repo).await.unwrap();
15128        state.enabled_worktree_config = true;
15129        state.status = RunStatus::Ready;
15130        state.candidates.push(Candidate {
15131            index: 0,
15132            label: 'A',
15133            agent: "implement".into(),
15134            branch: branch.clone(),
15135            worktree: candidate.clone(),
15136            summary: String::new(),
15137            stat: String::new(),
15138            files: 0,
15139            commits: 0,
15140            empty: false,
15141            failed: None,
15142            verified_noop: None,
15143            duration_ms: 0,
15144            folded: false,
15145        });
15146        state.save_under(&home).unwrap();
15147        let config_before = std::fs::read(common.join("config")).unwrap();
15148        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15149            .await
15150            .unwrap();
15151        // A separate OS handle bypasses the launcher's local gate, exactly as
15152        // an external updater does, and must still prevent all folding work.
15153        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15154        assert_eq!(
15155            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15156            0
15157        );
15158        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15159        assert!(
15160            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15161                .await
15162                .is_err()
15163        );
15164        assert!(candidate.exists());
15165        assert!(reviewer.exists());
15166        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15167        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15168        assert_eq!(
15169            git::git(&repo, &["worktree", "list", "--porcelain"])
15170                .await
15171                .unwrap(),
15172            worktrees_before
15173        );
15174        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15175            .await
15176            .unwrap()
15177            .unwrap_err();
15178        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15179        drop(fold);
15180        assert!(candidate.exists() && reviewer.exists());
15181        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15182        assert_eq!(
15183            state.resource_block.as_ref().unwrap().kind,
15184            "repository coordination"
15185        );
15186        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15187        drop(external);
15188        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15189        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15190            .await
15191            .unwrap()
15192            .unwrap();
15193        drop(fold);
15194        assert!(removed.contains(&branch));
15195        assert!(!root.exists());
15196        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15197        assert!(state.candidates[0].folded);
15198        assert!(!state.enabled_worktree_config);
15199        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15200        let enabled = git::git_raw(
15201            &repo,
15202            &["config", "--local", "--get", "extensions.worktreeConfig"],
15203        )
15204        .await
15205        .unwrap();
15206        assert!(!enabled.ok(), "the last config reference was released");
15207    }
15208
15209    /// A branch handed to a later run is that run's (and its pull request's):
15210    /// folding the run that released it must not delete it.
15211    #[tokio::test]
15212    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15213        let tmp = tempfile::tempdir().expect("tempdir");
15214        let repo = tmp.path().join("repo");
15215        std::fs::create_dir_all(&repo).unwrap();
15216        init_repo(&repo);
15217        let home = tmp.path().join("home");
15218
15219        let mut config = Config::default();
15220        config.graph.worktree_root = Some(tmp.path().join("wt"));
15221        let mut state = RunState::new(
15222            repo.clone(),
15223            "main".to_owned(),
15224            "deadbeef".to_owned(),
15225            "task".to_owned(),
15226            config,
15227        );
15228        // The worktree is already gone (released); the branch survives.
15229        git::git(&repo, &["branch", "magi/x/A", "main"])
15230            .await
15231            .expect("branch");
15232        state.candidates = vec![Candidate {
15233            index: 0,
15234            label: 'A',
15235            agent: "alpha".to_owned(),
15236            branch: "magi/x/A".to_owned(),
15237            worktree: state.worktree_root().join("cand-A"),
15238            summary: String::new(),
15239            stat: String::new(),
15240            files: 0,
15241            commits: 0,
15242            empty: false,
15243            failed: None,
15244            verified_noop: None,
15245            duration_ms: 0,
15246            folded: true,
15247        }];
15248        state.released_to = Some("20260901-000000-new1".to_owned());
15249        state.released_branches = vec!["magi/x/A".to_owned()];
15250
15251        fold_run(&mut state, true, &home).await.expect("fold_run");
15252        state.status = RunStatus::Merged;
15253        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15254        state.merge = Some(crate::run::MergeOutcome {
15255            mode: MergeMode::Pr,
15256            ok: true,
15257            detail: "confirmed merged".into(),
15258            empty: false,
15259        });
15260        fold_run(&mut state, true, &home)
15261            .await
15262            .expect("merged fold");
15263
15264        assert!(
15265            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15266            "the handed-over branch survives a fold"
15267        );
15268    }
15269
15270    /// A winner with nothing ahead of the base is caught before `gh` is ever
15271    /// asked for a pull request, and the message carries what the task's
15272    /// references resolved to.
15273    #[tokio::test]
15274    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15275        let tmp = tempfile::tempdir().expect("tempdir");
15276        let repo = tmp.path().join("repo");
15277        std::fs::create_dir_all(&repo).unwrap();
15278        init_repo(&repo);
15279        let run = |args: &[&str]| {
15280            let out = std::process::Command::new("git")
15281                .quiet()
15282                .args(args)
15283                .current_dir(&repo)
15284                .output()
15285                .expect("spawn git");
15286            assert!(out.status.success(), "git {args:?}");
15287        };
15288        run(&["branch", "magi/x/A"]);
15289        run(&["checkout", "-q", "-b", "magi/x/B"]);
15290        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15291        run(&["add", "-A"]);
15292        run(&["commit", "-q", "-m", "work"]);
15293        run(&["checkout", "-q", "main"]);
15294
15295        // A pull request is compared against the remote's base, so the
15296        // fixture needs one. Before it exists nothing can be read, and the
15297        // answer must be "not empty".
15298        let probe = RunState::new(
15299            repo.clone(),
15300            "main".to_owned(),
15301            "deadbeef".to_owned(),
15302            "task".to_owned(),
15303            Config::default(),
15304        );
15305        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15306        let bare = tmp.path().join("origin.git");
15307        let out = std::process::Command::new("git")
15308            .quiet()
15309            .args(["init", "-q", "--bare"])
15310            .arg(&bare)
15311            .output()
15312            .expect("spawn git");
15313        assert!(out.status.success(), "git init --bare");
15314        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15315        run(&["push", "-q", "origin", "main"]);
15316
15317        let mut state = RunState::new(
15318            repo.clone(),
15319            "main".to_owned(),
15320            "deadbeef".to_owned(),
15321            "task".to_owned(),
15322            Config::default(),
15323        );
15324        state.seeds = vec![refs::Seed {
15325            token: "magi/27b2/A".to_owned(),
15326            kind: refs::SeedKind::Unresolved,
15327            sha: String::new(),
15328            branch: true,
15329            detail: "no branch or commit named magi/27b2/A".to_owned(),
15330        }];
15331
15332        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15333        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15334        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15335        let detail = empty_candidate_detail(&state, "main");
15336        assert!(detail.starts_with("empty candidate"), "{detail}");
15337        assert!(!detail.contains("magi/27b2/A"), "{detail}");
15338        assert!(detail.contains("--start-from"), "{detail}");
15339    }
15340
15341    /// `status == Ready` used to be read as "this is the harmless
15342    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15343    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15344    /// run's PR was closed without merging — and reentering `merge` with
15345    /// `mode` still `Pr` does not know the difference, so it pushed and
15346    /// opened a second pull request. `mode == Local` reproduces the same
15347    /// blind spot without a network call: reentry must not attempt another
15348    /// git merge once this node has already recorded an outcome.
15349    #[tokio::test]
15350    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15351        let tmp = tempfile::tempdir().expect("tempdir");
15352        let repo = tmp.path().join("repo");
15353        std::fs::create_dir_all(&repo).unwrap();
15354        init_repo(&repo);
15355
15356        let mut config = Config::default();
15357        config.merge.mode = MergeMode::Local;
15358
15359        let mut state = RunState::new(
15360            repo.clone(),
15361            "main".to_owned(),
15362            "deadbeef".to_owned(),
15363            "task".to_owned(),
15364            config,
15365        );
15366        state.candidates = vec![Candidate {
15367            index: 0,
15368            label: 'A',
15369            agent: "alpha".to_owned(),
15370            branch: "does-not-exist".to_owned(),
15371            worktree: repo.clone(),
15372            summary: String::new(),
15373            stat: String::new(),
15374            files: 0,
15375            commits: 0,
15376            empty: false,
15377            failed: None,
15378            verified_noop: None,
15379            duration_ms: 0,
15380            folded: false,
15381        }];
15382        state.tally = Some(Tally {
15383            first_choice: BTreeMap::from([('A', 1)]),
15384            borda: BTreeMap::new(),
15385            winner: 'A',
15386            rankings: 1,
15387            unanimous_initial: true,
15388            deliberated: false,
15389            changed_votes: 0,
15390            unanimous_final: true,
15391            tie_break: None,
15392            judges: 0,
15393            present: 0,
15394            quorum: 0,
15395            met_quorum: true,
15396            uncontested: Some("only candidate A produced a change".to_owned()),
15397        });
15398        state.reviews = vec![ReviewRound {
15399            round: 1,
15400            head: "deadbeef".to_owned(),
15401            verified_head: None,
15402            verified_at: None,
15403            reviews: Vec::new(),
15404            e2e: Vec::new(),
15405            fix: None,
15406            blocking: 0,
15407            answered: 0,
15408            expected: 0,
15409            clean: true,
15410            verify_retried: false,
15411            e2e_deferred: false,
15412            e2e_defer_reason: None,
15413            progressed: false,
15414            vote_split: false,
15415            reconsideration: Vec::new(),
15416            verdict: None,
15417        }];
15418        state.gate = vec![CommandOutcome {
15419            command: "test".to_owned(),
15420            code: Some(0),
15421            output_tail: String::new(),
15422            duration_ms: 0,
15423            resource_blocked: false,
15424        }];
15425        state.gate_ran = true;
15426        // Reached its conclusion already — e.g. `land` closing the PR without
15427        // merging it, which (like the honest `MergeMode::None` path) leaves
15428        // `status` at `Ready`. The recorded outcome is what actually marks
15429        // this node done.
15430        state.status = RunStatus::Ready;
15431        state.merge = Some(MergeOutcome {
15432            mode: MergeMode::Local,
15433            ok: false,
15434            detail: "already concluded".to_owned(),
15435            empty: false,
15436        });
15437
15438        let mut runner = Runner {
15439            state,
15440            roles: ResolvedRoles {
15441                implementers: Vec::new(),
15442                judges: Vec::new(),
15443                reviewers: Vec::new(),
15444                fixer: None,
15445                conductor: conductor(),
15446                implementer_roster: Vec::new(),
15447                judge_roster: Vec::new(),
15448                reviewer_roster: Vec::new(),
15449            },
15450            sem: Arc::new(Semaphore::new(1)),
15451            pause: Pause::new(),
15452            interrupt: Pause::new(),
15453        };
15454
15455        runner.merge().await.expect("merge");
15456
15457        assert_eq!(
15458            runner.state.status,
15459            RunStatus::Ready,
15460            "a concluded run's status must not change on reentry"
15461        );
15462        assert_eq!(
15463            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15464            Some("already concluded"),
15465            "merge must not run again once the node already recorded an outcome"
15466        );
15467    }
15468
15469    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15470    /// when its last attempt was resource-blocked (the shared build cache
15471    /// could not be acquired or confirmed fresh in time - see
15472    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15473    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15474    /// a run merge a tree the gate never actually checked - exactly the case
15475    /// a contended cache produces on every retry until it clears. `merge`
15476    /// must refuse until `gate` has actually recorded an attempt.
15477    #[tokio::test]
15478    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15479        let tmp = tempfile::tempdir().expect("tempdir");
15480        let repo = tmp.path().join("repo");
15481        std::fs::create_dir_all(&repo).unwrap();
15482        init_repo(&repo);
15483
15484        let mut config = Config::default();
15485        config.merge.mode = MergeMode::Local;
15486
15487        let mut state = RunState::new(
15488            repo.clone(),
15489            "main".to_owned(),
15490            "deadbeef".to_owned(),
15491            "task".to_owned(),
15492            config,
15493        );
15494        state.candidates = vec![Candidate {
15495            index: 0,
15496            label: 'A',
15497            agent: "alpha".to_owned(),
15498            branch: "does-not-exist".to_owned(),
15499            worktree: repo.clone(),
15500            summary: String::new(),
15501            stat: String::new(),
15502            files: 0,
15503            commits: 0,
15504            empty: false,
15505            failed: None,
15506            verified_noop: None,
15507            duration_ms: 0,
15508            folded: false,
15509        }];
15510        state.tally = Some(Tally {
15511            first_choice: BTreeMap::from([('A', 1)]),
15512            borda: BTreeMap::new(),
15513            winner: 'A',
15514            rankings: 1,
15515            unanimous_initial: true,
15516            deliberated: false,
15517            changed_votes: 0,
15518            unanimous_final: true,
15519            tie_break: None,
15520            judges: 0,
15521            present: 0,
15522            quorum: 0,
15523            met_quorum: true,
15524            uncontested: Some("only candidate A produced a change".to_owned()),
15525        });
15526        state.reviews = vec![ReviewRound {
15527            round: 1,
15528            head: "deadbeef".to_owned(),
15529            verified_head: None,
15530            verified_at: None,
15531            reviews: Vec::new(),
15532            e2e: Vec::new(),
15533            fix: None,
15534            blocking: 0,
15535            answered: 0,
15536            expected: 0,
15537            clean: true,
15538            verify_retried: false,
15539            e2e_deferred: false,
15540            e2e_defer_reason: None,
15541            progressed: false,
15542            vote_split: false,
15543            reconsideration: Vec::new(),
15544            verdict: None,
15545        }];
15546        // The point: `gate` has not recorded anything yet.
15547        state.gate = Vec::new();
15548        state.gate_ran = false;
15549        state.status = RunStatus::Gating;
15550
15551        let mut runner = Runner {
15552            state,
15553            roles: ResolvedRoles {
15554                implementers: Vec::new(),
15555                judges: Vec::new(),
15556                reviewers: Vec::new(),
15557                fixer: None,
15558                conductor: conductor(),
15559                implementer_roster: Vec::new(),
15560                judge_roster: Vec::new(),
15561                reviewer_roster: Vec::new(),
15562            },
15563            sem: Arc::new(Semaphore::new(1)),
15564            pause: Pause::new(),
15565            interrupt: Pause::new(),
15566        };
15567
15568        runner.merge().await.expect("merge");
15569
15570        assert!(
15571            runner.state.merge.is_none(),
15572            "an empty gate must never be read as a passing one: {:?}",
15573            runner.state.merge
15574        );
15575    }
15576
15577    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
15578    /// commands configured and `merge.mode` is `none` (a review-only run).
15579    /// `gate` must still record a real attempt — zero commands, vacuously
15580    /// passed — rather than leaving `state.gate` empty in a way `merge`
15581    /// cannot tell apart from "never ran"; otherwise the run reaches
15582    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
15583    #[tokio::test]
15584    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15585        let tmp = tempfile::tempdir().expect("tempdir");
15586        let repo = tmp.path().join("repo");
15587        std::fs::create_dir_all(&repo).unwrap();
15588        init_repo(&repo);
15589
15590        // Default config: `verify.gate` empty, `merge.mode` is `none`.
15591        let config = Config::default();
15592
15593        let mut state = RunState::new(
15594            repo.clone(),
15595            "main".to_owned(),
15596            "deadbeef".to_owned(),
15597            "task".to_owned(),
15598            config,
15599        );
15600        state.candidates = vec![Candidate {
15601            index: 0,
15602            label: 'A',
15603            agent: "alpha".to_owned(),
15604            branch: "does-not-exist".to_owned(),
15605            worktree: repo.clone(),
15606            summary: String::new(),
15607            stat: String::new(),
15608            files: 0,
15609            commits: 0,
15610            empty: false,
15611            failed: None,
15612            verified_noop: None,
15613            duration_ms: 0,
15614            folded: false,
15615        }];
15616        state.tally = Some(Tally {
15617            first_choice: BTreeMap::from([('A', 1)]),
15618            borda: BTreeMap::new(),
15619            winner: 'A',
15620            rankings: 1,
15621            unanimous_initial: true,
15622            deliberated: false,
15623            changed_votes: 0,
15624            unanimous_final: true,
15625            tie_break: None,
15626            judges: 0,
15627            present: 0,
15628            quorum: 0,
15629            met_quorum: true,
15630            uncontested: Some("only candidate A produced a change".to_owned()),
15631        });
15632        state.reviews = vec![ReviewRound {
15633            round: 1,
15634            head: "deadbeef".to_owned(),
15635            verified_head: None,
15636            verified_at: None,
15637            reviews: Vec::new(),
15638            e2e: Vec::new(),
15639            fix: None,
15640            blocking: 0,
15641            answered: 0,
15642            expected: 0,
15643            clean: true,
15644            verify_retried: false,
15645            e2e_deferred: false,
15646            e2e_defer_reason: None,
15647            progressed: false,
15648            vote_split: false,
15649            reconsideration: Vec::new(),
15650            verdict: None,
15651        }];
15652
15653        let mut runner = Runner {
15654            state,
15655            roles: ResolvedRoles {
15656                implementers: Vec::new(),
15657                judges: Vec::new(),
15658                reviewers: Vec::new(),
15659                fixer: None,
15660                conductor: conductor(),
15661                implementer_roster: Vec::new(),
15662                judge_roster: Vec::new(),
15663                reviewer_roster: Vec::new(),
15664            },
15665            sem: Arc::new(Semaphore::new(1)),
15666            pause: Pause::new(),
15667            interrupt: Pause::new(),
15668        };
15669
15670        runner.gate().await.expect("gate");
15671        assert!(
15672            runner.state.gate_ran,
15673            "zero configured commands is still a real attempt, not an unrun gate"
15674        );
15675        assert!(runner.state.gate.is_empty());
15676        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15677        assert_ne!(
15678            runner.state.status,
15679            RunStatus::Blocked,
15680            "a gate with nothing to check must not read as failed"
15681        );
15682
15683        runner.merge().await.expect("merge");
15684        assert_eq!(
15685            runner.state.status,
15686            RunStatus::Ready,
15687            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15688        );
15689    }
15690
15691    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15692    /// `verify.gate` (so the e2e leg and the final gate never build against
15693    /// different directories). With zero `verify.gate` commands but a
15694    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15695    /// that lease before discovering it had nothing to run - so a repo with
15696    /// no gate commands could come back `resource_blocked` (and therefore
15697    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15698    /// cache, exactly the contention this run's own zero commands could
15699    /// never have touched. `gate` must recognise there is nothing to check
15700    /// before it ever asks for the lease.
15701    #[tokio::test]
15702    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15703        crate::run::pin_test_home();
15704        let home = crate::run::home();
15705
15706        let tmp = tempfile::tempdir().expect("tempdir");
15707        let repo = tmp.path().join("repo");
15708        std::fs::create_dir_all(&repo).unwrap();
15709        init_repo(&repo);
15710        // Unique to this test, so holding its lease cannot collide with
15711        // another test sharing the same process-wide `home`.
15712        let cache_dir = tmp.path().join("target");
15713
15714        let mut config = Config::default();
15715        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15716        // `verify.gate` stays empty (the default). Bounded so a regression
15717        // that does start waiting fails the test in seconds, not hangs it.
15718        config.graph.timeout_verify = Some(2);
15719        config.graph.timeout_resource_wait = 2;
15720
15721        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15722        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15723            .expect("no io error acquiring directly")
15724        {
15725            crate::cache::AcquireOutcome::Acquired(g) => g,
15726            crate::cache::AcquireOutcome::Busy(b) => {
15727                panic!("expected the direct acquire to win the lease first: {b:?}")
15728            }
15729        };
15730
15731        let mut state = RunState::new(
15732            repo.clone(),
15733            "main".to_owned(),
15734            "deadbeef".to_owned(),
15735            "task".to_owned(),
15736            config,
15737        );
15738        state.candidates = vec![Candidate {
15739            index: 0,
15740            label: 'A',
15741            agent: "alpha".to_owned(),
15742            branch: "does-not-exist".to_owned(),
15743            worktree: repo.clone(),
15744            summary: String::new(),
15745            stat: String::new(),
15746            files: 0,
15747            commits: 0,
15748            empty: false,
15749            failed: None,
15750            verified_noop: None,
15751            duration_ms: 0,
15752            folded: false,
15753        }];
15754        state.tally = Some(Tally {
15755            first_choice: BTreeMap::from([('A', 1)]),
15756            borda: BTreeMap::new(),
15757            winner: 'A',
15758            rankings: 1,
15759            unanimous_initial: true,
15760            deliberated: false,
15761            changed_votes: 0,
15762            unanimous_final: true,
15763            tie_break: None,
15764            judges: 0,
15765            present: 0,
15766            quorum: 0,
15767            met_quorum: true,
15768            uncontested: Some("only candidate A produced a change".to_owned()),
15769        });
15770        state.reviews = vec![ReviewRound {
15771            round: 1,
15772            head: "deadbeef".to_owned(),
15773            verified_head: None,
15774            verified_at: None,
15775            reviews: Vec::new(),
15776            e2e: Vec::new(),
15777            fix: None,
15778            blocking: 0,
15779            answered: 0,
15780            expected: 0,
15781            clean: true,
15782            verify_retried: false,
15783            e2e_deferred: false,
15784            e2e_defer_reason: None,
15785            progressed: false,
15786            vote_split: false,
15787            reconsideration: Vec::new(),
15788            verdict: None,
15789        }];
15790
15791        let mut runner = Runner {
15792            state,
15793            roles: ResolvedRoles {
15794                implementers: Vec::new(),
15795                judges: Vec::new(),
15796                reviewers: Vec::new(),
15797                fixer: None,
15798                conductor: conductor(),
15799                implementer_roster: Vec::new(),
15800                judge_roster: Vec::new(),
15801                reviewer_roster: Vec::new(),
15802            },
15803            sem: Arc::new(Semaphore::new(1)),
15804            pause: Pause::new(),
15805            interrupt: Pause::new(),
15806        };
15807
15808        let started = std::time::Instant::now();
15809        runner.gate().await.expect("gate");
15810        assert!(
15811            started.elapsed() < Duration::from_secs(1),
15812            "a gate with nothing to run must never wait on a lease it never needed"
15813        );
15814        assert!(
15815            runner.state.gate_ran,
15816            "zero commands is still a real, immediate attempt"
15817        );
15818        assert!(runner.state.gate.is_empty());
15819        assert_ne!(
15820            runner.state.status,
15821            RunStatus::Blocked,
15822            "must not read as resource-blocked on a lease it never asked for"
15823        );
15824    }
15825
15826    #[cfg(unix)]
15827    #[tokio::test]
15828    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15829        crate::run::pin_test_home();
15830        let tmp = tempfile::tempdir().unwrap();
15831        let mut config = Config::default();
15832        config.disk.min_free_bytes = 0;
15833        let mut state = RunState::new(
15834            tmp.path().to_owned(),
15835            "main".into(),
15836            "base".into(),
15837            "verification evidence".into(),
15838            config,
15839        );
15840        let (outcomes, pids) = run_commands(
15841            &mut state,
15842            "gate",
15843            "gate",
15844            1,
15845            &["sh".into(), "-c".into()],
15846            &[
15847                "printf before-timeout; exec sleep 30".into(),
15848                "kill -TERM $$".into(),
15849                "exit 7".into(),
15850                "printf success".into(),
15851            ],
15852            tmp.path(),
15853            Duration::from_secs(1),
15854        )
15855        .await;
15856        wait_for_timed_out_children_to_die(&pids).await;
15857        assert_eq!(
15858            pids.len(),
15859            1,
15860            "unleased timeout retains its death-check PID"
15861        );
15862        assert_eq!(outcomes.len(), 4);
15863        assert_eq!(outcomes[0].code, None);
15864        assert!(outcomes[0].output_tail.contains("before-timeout"));
15865        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15866        assert_eq!(outcomes[1].code, None);
15867        assert!(outcomes[1].output_tail.contains("command terminated:"));
15868        assert!(!outcomes[1].output_tail.contains("timed out"));
15869        assert_eq!(outcomes[2].code, Some(7));
15870        assert!(!outcomes[2].output_tail.contains("timed out"));
15871        assert_eq!(outcomes[3].code, Some(0));
15872        assert_eq!(outcomes[3].output_tail, "success");
15873        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15874    }
15875
15876    /// The addendum's second gap: a `verify.gate` command running for real
15877    /// wall-clock time had nothing at all to show for it in `active` before
15878    /// `run_commands` learned to record it — a run could sit in `Gating` for
15879    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
15880    /// what was actually happening. Proven with a genuinely still-running
15881    /// command, not just a before/after check on the final state: a poller
15882    /// task reads the same `run.json` `gate()` is writing, the same way the
15883    /// phone or `magi show` would, while the shell command is still blocked
15884    /// on its own release marker.
15885    #[tokio::test]
15886    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15887        crate::run::pin_test_home();
15888
15889        let tmp = tempfile::tempdir().expect("tempdir");
15890        let repo = tmp.path().join("repo");
15891        std::fs::create_dir_all(&repo).unwrap();
15892        init_repo(&repo);
15893
15894        let mut config = Config::default();
15895        config.verify.gate = vec![
15896            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15897             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15898                .to_owned(),
15899        ];
15900
15901        let mut state = RunState::new(
15902            repo.clone(),
15903            "main".to_owned(),
15904            "deadbeef".to_owned(),
15905            "task".to_owned(),
15906            config,
15907        );
15908        let run_id = state.id.clone();
15909        state.candidates = vec![Candidate {
15910            index: 0,
15911            label: 'A',
15912            agent: "alpha".to_owned(),
15913            branch: "does-not-exist".to_owned(),
15914            worktree: repo.clone(),
15915            summary: String::new(),
15916            stat: String::new(),
15917            files: 0,
15918            commits: 0,
15919            empty: false,
15920            failed: None,
15921            verified_noop: None,
15922            duration_ms: 0,
15923            folded: false,
15924        }];
15925        state.tally = Some(Tally {
15926            first_choice: BTreeMap::from([('A', 1)]),
15927            borda: BTreeMap::new(),
15928            winner: 'A',
15929            rankings: 1,
15930            unanimous_initial: true,
15931            deliberated: false,
15932            changed_votes: 0,
15933            unanimous_final: true,
15934            tie_break: None,
15935            judges: 0,
15936            present: 0,
15937            quorum: 0,
15938            met_quorum: true,
15939            uncontested: Some("only candidate A produced a change".to_owned()),
15940        });
15941        state.reviews = vec![ReviewRound {
15942            round: 1,
15943            head: "deadbeef".to_owned(),
15944            verified_head: None,
15945            verified_at: None,
15946            reviews: Vec::new(),
15947            e2e: Vec::new(),
15948            fix: None,
15949            blocking: 0,
15950            answered: 0,
15951            expected: 0,
15952            clean: true,
15953            verify_retried: false,
15954            e2e_deferred: false,
15955            e2e_defer_reason: None,
15956            progressed: false,
15957            vote_split: false,
15958            reconsideration: Vec::new(),
15959            verdict: None,
15960        }];
15961
15962        let mut runner = Runner {
15963            state,
15964            roles: ResolvedRoles {
15965                implementers: Vec::new(),
15966                judges: Vec::new(),
15967                reviewers: Vec::new(),
15968                fixer: None,
15969                conductor: conductor(),
15970                implementer_roster: Vec::new(),
15971                judge_roster: Vec::new(),
15972                reviewer_roster: Vec::new(),
15973            },
15974            sem: Arc::new(Semaphore::new(1)),
15975            pause: Pause::new(),
15976            interrupt: Pause::new(),
15977        };
15978
15979        let started_marker = repo.join("started.marker");
15980        let release_marker = repo.join("release.marker");
15981        let poller = tokio::spawn(async move {
15982            // Bounded so a regression that never records the task entry
15983            // fails this test in seconds instead of hanging the suite —
15984            // the same shape `a_park_requested_while_a_seat_is_mid_call_
15985            // does_not_cut_it_short` uses for the same reason.
15986            for _ in 0..100 {
15987                if started_marker.exists()
15988                    && let Ok(s) = crate::run::RunState::load(&run_id)
15989                    && let Some(a) = s.active.get("gate")
15990                {
15991                    std::fs::write(&release_marker, b"go").expect("release marker");
15992                    return Some(a.clone());
15993                }
15994                tokio::time::sleep(Duration::from_millis(50)).await;
15995            }
15996            None
15997        });
15998
15999        runner.gate().await.expect("gate");
16000        let captured = poller.await.expect("poller task");
16001        let captured = captured.expect(
16002            "the poller never saw a `gate` task entry in run.json while the command was \
16003             still blocked on its own release marker",
16004        );
16005
16006        assert_eq!(captured.task.as_deref(), Some("gate"));
16007        assert_eq!(captured.node, "gate");
16008        assert_eq!(captured.index, Some(1));
16009        assert_eq!(captured.total, Some(1));
16010        assert!(
16011            captured
16012                .command
16013                .as_deref()
16014                .is_some_and(|c| c.contains("started.marker")),
16015            "{captured:?}"
16016        );
16017
16018        assert!(
16019            runner.state.active.is_empty(),
16020            "the entry must be cleared once the command actually finished: {:?}",
16021            runner.state.active
16022        );
16023        assert!(runner.state.gate_ran);
16024        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16025    }
16026
16027    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16028    /// record for `land`; one with only a Minor, or no reject, leaves none.
16029    #[tokio::test]
16030    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16031        use crate::verdict::{Finding, ReviewVote, Severity};
16032        crate::run::pin_test_home();
16033        let tmp = tempfile::tempdir().expect("tempdir");
16034        let repo = tmp.path().join("repo");
16035        std::fs::create_dir_all(&repo).unwrap();
16036        init_repo(&repo);
16037
16038        for (severity, vote, expect) in [
16039            (Severity::Major, ReviewVote::Reject, true),
16040            (Severity::Minor, ReviewVote::Reject, false),
16041            (Severity::Major, ReviewVote::Approve, false),
16042        ] {
16043            let mut round = review_round(false, 1, 1, 1, false, true);
16044            round.reviews = vec![ReviewRecord {
16045                reviewer: 1,
16046                agent: "alpha".to_owned(),
16047                summary: String::new(),
16048                findings: vec![Finding {
16049                    id: "R1-1-1".to_owned(),
16050                    severity,
16051                    file: None,
16052                    line: None,
16053                    title: "t".to_owned(),
16054                    detail: String::new(),
16055                }],
16056                vote: Some(vote),
16057                failed: None,
16058                duration_ms: 0,
16059                attempts: 0,
16060            }];
16061            let mut state = RunState::new(
16062                repo.clone(),
16063                "main".to_owned(),
16064                "deadbeef".to_owned(),
16065                "task".to_owned(),
16066                Config::default(),
16067            );
16068            state.reviews = vec![round];
16069            let mut runner = Runner {
16070                state,
16071                roles: ResolvedRoles {
16072                    implementers: Vec::new(),
16073                    judges: Vec::new(),
16074                    reviewers: Vec::new(),
16075                    fixer: None,
16076                    conductor: conductor(),
16077                    implementer_roster: Vec::new(),
16078                    judge_roster: Vec::new(),
16079                    reviewer_roster: Vec::new(),
16080                },
16081                sem: Arc::new(Semaphore::new(1)),
16082                pause: Pause::new(),
16083                interrupt: Pause::new(),
16084            };
16085            let shell = runner.state.config.shell();
16086            runner
16087                .stop_reviewing("round budget spent", &shell, &repo)
16088                .await
16089                .expect("stop_reviewing");
16090            assert_eq!(runner.state.status, RunStatus::Gating);
16091            assert_eq!(
16092                runner.state.contested_handoff.is_some(),
16093                expect,
16094                "{severity:?} + {vote:?}"
16095            );
16096        }
16097    }
16098
16099    /// The shape the incident this whole fix responds to actually had: the
16100    /// round budget spent, the last round's own e2e blocked on the shared
16101    /// build cache (held here by a live pid — this test process — exactly
16102    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16103    /// without forking a process). `stop_reviewing` must retry it — not
16104    /// silently leave the round looking untouched (the catch-up-only half of
16105    /// the bug), and not read the contention as a red `e2e` and block the
16106    /// run on it (the other half). Called directly, the same way
16107    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16108    /// above exercises `gate`, so this never needs a real cargo build to
16109    /// reach: the lease is never released, so `with_cache_lease` never gets
16110    /// past acquiring it into anything that would need a real workspace.
16111    #[tokio::test]
16112    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16113        crate::run::pin_test_home();
16114        let home = crate::run::home();
16115
16116        let tmp = tempfile::tempdir().expect("tempdir");
16117        let repo = tmp.path().join("repo");
16118        std::fs::create_dir_all(&repo).unwrap();
16119        init_repo(&repo);
16120        let head = crate::git::rev_parse(&repo, "HEAD")
16121            .await
16122            .expect("rev-parse");
16123        // Unique to this test, so holding its lease cannot collide with
16124        // another test sharing the same process-wide `home`.
16125        let cache_dir = tmp.path().join("target");
16126
16127        let mut config = Config::default();
16128        config.verify.e2e = vec![format!(
16129            "CARGO_TARGET_DIR='{}' test -f README.md",
16130            cache_dir.display()
16131        )];
16132        config.graph.review_rounds = 1;
16133        // Bounded so a regression that does start waiting fails the test in
16134        // seconds, not hangs it.
16135        config.graph.timeout_verify = Some(2);
16136        config.graph.timeout_resource_wait = 2;
16137
16138        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16139        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16140            .expect("no io error acquiring directly")
16141        {
16142            crate::cache::AcquireOutcome::Acquired(g) => g,
16143            crate::cache::AcquireOutcome::Busy(b) => {
16144                panic!("expected the direct acquire to win the lease first: {b:?}")
16145            }
16146        };
16147
16148        let mut state = RunState::new(
16149            repo.clone(),
16150            "main".to_owned(),
16151            head.clone(),
16152            "task".to_owned(),
16153            config,
16154        );
16155        state.candidates = vec![Candidate {
16156            index: 0,
16157            label: 'A',
16158            agent: "alpha".to_owned(),
16159            branch: "does-not-exist".to_owned(),
16160            worktree: repo.clone(),
16161            summary: String::new(),
16162            stat: String::new(),
16163            files: 0,
16164            commits: 0,
16165            empty: false,
16166            failed: None,
16167            verified_noop: None,
16168            duration_ms: 0,
16169            folded: false,
16170        }];
16171        state.tally = Some(Tally {
16172            first_choice: BTreeMap::from([('A', 1)]),
16173            borda: BTreeMap::new(),
16174            winner: 'A',
16175            rankings: 1,
16176            unanimous_initial: true,
16177            deliberated: false,
16178            changed_votes: 0,
16179            unanimous_final: true,
16180            tie_break: None,
16181            judges: 0,
16182            present: 0,
16183            quorum: 0,
16184            met_quorum: true,
16185            uncontested: Some("only candidate A produced a change".to_owned()),
16186        });
16187        // The round budget's last round, deferred: `needs_catchup_run`'s
16188        // other trigger. `stop_reviewing`'s retry machinery must treat this
16189        // exactly like a resource-blocked attempt once it actually runs.
16190        state.reviews = vec![ReviewRound {
16191            round: 1,
16192            head: head.clone(),
16193            verified_head: None,
16194            verified_at: None,
16195            reviews: Vec::new(),
16196            e2e: Vec::new(),
16197            fix: None,
16198            blocking: 1,
16199            answered: 1,
16200            expected: 1,
16201            clean: false,
16202            verify_retried: false,
16203            e2e_deferred: true,
16204            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16205            progressed: false,
16206            vote_split: false,
16207            reconsideration: Vec::new(),
16208            verdict: None,
16209        }];
16210
16211        let mut runner = Runner {
16212            state,
16213            roles: ResolvedRoles {
16214                implementers: Vec::new(),
16215                judges: Vec::new(),
16216                reviewers: Vec::new(),
16217                fixer: None,
16218                conductor: conductor(),
16219                implementer_roster: Vec::new(),
16220                judge_roster: Vec::new(),
16221                reviewer_roster: Vec::new(),
16222            },
16223            sem: Arc::new(Semaphore::new(1)),
16224            pause: Pause::new(),
16225            interrupt: Pause::new(),
16226        };
16227
16228        let shell = runner.state.config.shell();
16229        runner
16230            .stop_reviewing("round budget spent", &shell, &repo)
16231            .await
16232            .expect("stop_reviewing");
16233
16234        let last = runner.state.reviews.last().expect("round record");
16235        assert_eq!(
16236            last.e2e_status(),
16237            E2eStatus::ResourceBlocked,
16238            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16239             failed: {last:?}"
16240        );
16241        assert_eq!(
16242            last.verified_head.as_deref(),
16243            Some(head.as_str()),
16244            "which commit this attempt targeted is known even though nothing finished checking \
16245             it"
16246        );
16247        let first_attempt_at = last
16248            .verified_at
16249            .expect("when this attempt ran is known too");
16250        assert_ne!(
16251            runner.state.status,
16252            RunStatus::Blocked,
16253            "contention is evidence about the machine, not the patch — it must not settle the \
16254             run as blocked: {:?}",
16255            runner.state.status
16256        );
16257        assert!(
16258            !runner
16259                .state
16260                .events
16261                .iter()
16262                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16263            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16264            runner.state.events
16265        );
16266
16267        // The cache is still held: a later reentry must retry the same
16268        // round's verification again — not leave it looking exactly as
16269        // untouched as the first blocked attempt, which is indistinguishable
16270        // from never having tried again at all.
16271        runner
16272            .stop_reviewing("round budget spent", &shell, &repo)
16273            .await
16274            .expect("stop_reviewing retry");
16275        assert_eq!(
16276            runner.state.reviews.len(),
16277            1,
16278            "no new round was started: {:?}",
16279            runner.state.reviews
16280        );
16281        let last = runner.state.reviews.last().expect("round record");
16282        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16283        assert!(
16284            last.verified_at.expect("still known") > first_attempt_at,
16285            "a second reentry must be a fresh attempt, not a stale copy of the first"
16286        );
16287        assert_ne!(runner.state.status, RunStatus::Blocked);
16288
16289        held.release();
16290    }
16291
16292    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
16293    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
16294    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
16295    /// own top-of-function fast path (`review_conclusion`) correctly reads
16296    /// this shape as `None` rather than guessing `Blocked`, and the loop's
16297    /// own `for` range is empty once the round budget is spent, so
16298    /// `review_loop` must retry the check itself rather than silently doing
16299    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
16300    /// above exercises directly, but through `review_loop`'s own entry point
16301    /// this time, proving the wiring between the two rather than just the
16302    /// retry logic in isolation.
16303    #[tokio::test]
16304    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16305        crate::run::pin_test_home();
16306        let home = crate::run::home();
16307
16308        let tmp = tempfile::tempdir().expect("tempdir");
16309        let repo = tmp.path().join("repo");
16310        std::fs::create_dir_all(&repo).unwrap();
16311        init_repo(&repo);
16312        let head = crate::git::rev_parse(&repo, "HEAD")
16313            .await
16314            .expect("rev-parse");
16315        let cache_dir = tmp.path().join("target");
16316
16317        let mut config = Config::default();
16318        config.verify.e2e = vec![format!(
16319            "CARGO_TARGET_DIR='{}' test -f README.md",
16320            cache_dir.display()
16321        )];
16322        config.graph.review_rounds = 1;
16323        config.graph.timeout_verify = Some(2);
16324        config.graph.timeout_resource_wait = 2;
16325
16326        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16327        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16328            .expect("no io error acquiring directly")
16329        {
16330            crate::cache::AcquireOutcome::Acquired(g) => g,
16331            crate::cache::AcquireOutcome::Busy(b) => {
16332                panic!("expected the direct acquire to win the lease first: {b:?}")
16333            }
16334        };
16335
16336        let mut state = RunState::new(
16337            repo.clone(),
16338            "main".to_owned(),
16339            head.clone(),
16340            "task".to_owned(),
16341            config,
16342        );
16343        state.candidates = vec![Candidate {
16344            index: 0,
16345            label: 'A',
16346            agent: "alpha".to_owned(),
16347            branch: "does-not-exist".to_owned(),
16348            worktree: repo.clone(),
16349            summary: String::new(),
16350            stat: String::new(),
16351            files: 0,
16352            commits: 0,
16353            empty: false,
16354            failed: None,
16355            verified_noop: None,
16356            duration_ms: 0,
16357            folded: false,
16358        }];
16359        state.tally = Some(Tally {
16360            first_choice: BTreeMap::from([('A', 1)]),
16361            borda: BTreeMap::new(),
16362            winner: 'A',
16363            rankings: 1,
16364            unanimous_initial: true,
16365            deliberated: false,
16366            changed_votes: 0,
16367            unanimous_final: true,
16368            tie_break: None,
16369            judges: 0,
16370            present: 0,
16371            quorum: 0,
16372            met_quorum: true,
16373            uncontested: Some("only candidate A produced a change".to_owned()),
16374        });
16375        // The exact shape a prior process's `stop_reviewing` would have left
16376        // on disk: the round budget's last round, a real attempt already
16377        // made and already resource-blocked.
16378        state.reviews = vec![ReviewRound {
16379            round: 1,
16380            head: head.clone(),
16381            verified_head: Some(head.clone()),
16382            verified_at: Some(jiff::Timestamp::now()),
16383            reviews: Vec::new(),
16384            e2e: vec![CommandOutcome {
16385                command: format!(
16386                    "CARGO_TARGET_DIR='{}' test -f README.md",
16387                    cache_dir.display()
16388                ),
16389                code: None,
16390                output_tail: "waiting for the shared build cache".to_owned(),
16391                duration_ms: 0,
16392                resource_blocked: true,
16393            }],
16394            fix: None,
16395            blocking: 1,
16396            answered: 1,
16397            expected: 1,
16398            clean: false,
16399            verify_retried: false,
16400            e2e_deferred: false,
16401            e2e_defer_reason: None,
16402            progressed: false,
16403            vote_split: false,
16404            reconsideration: Vec::new(),
16405            verdict: None,
16406        }];
16407
16408        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16409        let mut runner = Runner {
16410            state,
16411            roles: ResolvedRoles {
16412                implementers: Vec::new(),
16413                judges: Vec::new(),
16414                reviewers: Vec::new(),
16415                fixer: None,
16416                conductor: conductor(),
16417                implementer_roster: Vec::new(),
16418                judge_roster: Vec::new(),
16419                reviewer_roster: Vec::new(),
16420            },
16421            sem: Arc::new(Semaphore::new(1)),
16422            pause: Pause::new(),
16423            interrupt: Pause::new(),
16424        };
16425
16426        // The lease is still held throughout, so this reentry's own retry is
16427        // also contended — proving `review_loop` actually tried again (not
16428        // that it happened to succeed) is what the timestamp comparison
16429        // below is for.
16430        runner.review_loop().await.expect("review_loop");
16431
16432        assert_eq!(
16433            runner.state.reviews.len(),
16434            1,
16435            "no new round was started on top of the unresolved one: {:?}",
16436            runner.state.reviews
16437        );
16438        let last = &runner.state.reviews[0];
16439        assert_eq!(
16440            last.e2e_status(),
16441            E2eStatus::ResourceBlocked,
16442            "still contended: {last:?}"
16443        );
16444        assert!(
16445            last.verified_at.expect("still known") > first_attempt_at,
16446            "review_loop must have actually retried the check, not left it exactly as found"
16447        );
16448        assert_ne!(
16449            runner.state.status,
16450            RunStatus::Blocked,
16451            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16452            runner.state.status
16453        );
16454
16455        held.release();
16456    }
16457
16458    #[tokio::test]
16459    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16460        crate::run::pin_test_home();
16461        for status in [RunStatus::Blocked, RunStatus::Gating] {
16462            let mut runner = runner_at(status);
16463            runner.state.config.graph.land = true;
16464            runner.state.gate_ran = true;
16465            runner.state.merge = Some(MergeOutcome {
16466                mode: MergeMode::Pr,
16467                ok: false,
16468                detail: "the fixer produced no commit".into(),
16469                empty: false,
16470            });
16471            let url = "https://example.invalid/o/r/pull/1";
16472            runner.state.pr = Some(crate::run::PrRecord {
16473                url: url.into(),
16474                number: 1,
16475                state: "open".into(),
16476                checks: "red".into(),
16477                round: 1,
16478                rounds: 1,
16479                red_at_merge: Vec::new(),
16480            });
16481            runner.state.save().unwrap();
16482            // The nonexistent repository makes observation fail before a forge
16483            // command can run. We still must reach land, rather than silently
16484            // return from the graph with its old gate and merge records.
16485            runner.execute_graph().await.unwrap();
16486            assert_eq!(runner.state.status, RunStatus::Blocked);
16487            assert!(runner.state.land_sync_pending);
16488            assert!(runner.state.events.iter().any(|event|
16489                event.node == "land" && event.message == format!("watching {url}")));
16490            assert_eq!(
16491                runner.state.merge.as_ref().unwrap().detail,
16492                "the fixer produced no commit"
16493            );
16494        }
16495    }
16496
16497    #[tokio::test]
16498    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16499        crate::run::pin_test_home();
16500        let tmp = tempfile::tempdir().expect("tempdir");
16501        let repo = tmp.path().join("repo");
16502        std::fs::create_dir_all(&repo).unwrap();
16503        init_repo(&repo);
16504
16505        let mut config = Config::default();
16506        config.merge.mode = MergeMode::Pr;
16507        config.graph.land = true;
16508        config.graph.land_approval = false;
16509
16510        let mut state = RunState::new(
16511            repo.clone(),
16512            "main".to_owned(),
16513            "deadbeef".to_owned(),
16514            "task".to_owned(),
16515            config,
16516        );
16517        state.candidates = vec![Candidate {
16518            index: 0,
16519            label: 'A',
16520            agent: "alpha".to_owned(),
16521            branch: "does-not-exist".to_owned(),
16522            worktree: repo.clone(),
16523            summary: String::new(),
16524            stat: String::new(),
16525            files: 0,
16526            commits: 0,
16527            empty: false,
16528            failed: None,
16529            verified_noop: None,
16530            duration_ms: 0,
16531            folded: false,
16532        }];
16533        state.tally = Some(Tally {
16534            first_choice: BTreeMap::from([('A', 1)]),
16535            borda: BTreeMap::new(),
16536            winner: 'A',
16537            rankings: 1,
16538            unanimous_initial: true,
16539            deliberated: false,
16540            changed_votes: 0,
16541            unanimous_final: true,
16542            tie_break: None,
16543            judges: 0,
16544            present: 0,
16545            quorum: 0,
16546            met_quorum: true,
16547            uncontested: Some("only candidate A produced a change".to_owned()),
16548        });
16549        state.reviews = vec![ReviewRound {
16550            round: 1,
16551            head: "deadbeef".to_owned(),
16552            verified_head: None,
16553            verified_at: None,
16554            reviews: Vec::new(),
16555            e2e: Vec::new(),
16556            fix: None,
16557            blocking: 0,
16558            answered: 0,
16559            expected: 0,
16560            clean: true,
16561            verify_retried: false,
16562            e2e_deferred: false,
16563            e2e_defer_reason: None,
16564            progressed: false,
16565            vote_split: false,
16566            reconsideration: Vec::new(),
16567            verdict: None,
16568        }];
16569        state.gate = vec![CommandOutcome {
16570            command: "test".to_owned(),
16571            code: Some(0),
16572            output_tail: String::new(),
16573            duration_ms: 0,
16574            resource_blocked: false,
16575        }];
16576        state.gate_ran = true;
16577        // A first pass through `merge` already pushed and opened this pull
16578        // request; `status` is `Landing` because a previous call into `land`
16579        // parked or was interrupted before it reached a terminal outcome.
16580        state.status = RunStatus::Landing;
16581        state.merge = Some(MergeOutcome {
16582            mode: MergeMode::Pr,
16583            ok: true,
16584            detail: "https://example.invalid/x/y/pull/1".to_owned(),
16585            empty: false,
16586        });
16587
16588        // The Landing-resume shortcut calls `run_land` directly rather than
16589        // through `merge`, which is exactly the call site that used to skip
16590        // `settle_questions` - see the fixture below.
16591        ask_test_home();
16592        // A real resume loads a persisted run; driver ownership is checked
16593        // against that record before the Landing shortcut is entered.
16594        state.save().expect("save landing run");
16595        let store = ask::Questions::open();
16596        let q = ask_open_question(&store, &state.id);
16597
16598        let mut runner = Runner {
16599            state,
16600            roles: ResolvedRoles {
16601                implementers: Vec::new(),
16602                judges: Vec::new(),
16603                reviewers: Vec::new(),
16604                fixer: None,
16605                conductor: conductor(),
16606                implementer_roster: Vec::new(),
16607                judge_roster: Vec::new(),
16608                reviewer_roster: Vec::new(),
16609            },
16610            sem: Arc::new(Semaphore::new(1)),
16611            pause: Pause::new(),
16612            interrupt: Pause::new(),
16613        };
16614
16615        // `execute`, not `merge` directly: the Landing-resume shortcut lives
16616        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
16617        // exactly because `review_loop` would otherwise clobber the marker
16618        // first.
16619        runner.execute().await.expect("execute");
16620
16621        assert_eq!(
16622            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16623            Some("https://example.invalid/x/y/pull/1"),
16624            "reentry must not push again or open a second pull request over the \
16625             one `land` is already watching"
16626        );
16627        assert_ne!(
16628            runner.state.status,
16629            RunStatus::Landing,
16630            "land could not actually reach the fake pull request, so it must \
16631             have given up rather than left the run silently parked forever"
16632        );
16633        // `land` could not reach the fake pull request, so it gave up into
16634        // `Blocked` - still resumable, so the question must not have been
16635        // swept just because this branch now also calls `settle_questions`.
16636        assert_eq!(runner.state.status, RunStatus::Blocked);
16637        assert!(
16638            store.get(&q.id).unwrap().status.open(),
16639            "Blocked is still alive; settle_questions must have been a no-op here"
16640        );
16641    }
16642
16643    fn state_with_round(round: ReviewRound) -> RunState {
16644        let mut s = RunState::new(
16645            PathBuf::from("/repo"),
16646            "main".to_owned(),
16647            "abc1234".to_owned(),
16648            "add retries".to_owned(),
16649            Config::default(),
16650        );
16651        s.reviews = vec![round];
16652        s
16653    }
16654
16655    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16656        crate::verdict::Finding {
16657            id: id.to_owned(),
16658            severity,
16659            file: None,
16660            line: None,
16661            title: title.to_owned(),
16662            detail: String::new(),
16663        }
16664    }
16665
16666    #[test]
16667    fn pr_body_names_open_findings_and_declined_ones() {
16668        let round = ReviewRound {
16669            round: 2,
16670            head: "deadbee".to_owned(),
16671            verified_head: None,
16672            verified_at: None,
16673            reviews: vec![ReviewRecord {
16674                attempts: 0,
16675                reviewer: 1,
16676                agent: "alpha".to_owned(),
16677                summary: String::new(),
16678                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16679                vote: None,
16680                failed: None,
16681                duration_ms: 0,
16682            }],
16683            e2e: vec![CommandOutcome {
16684                command: "cargo test".to_owned(),
16685                code: Some(0),
16686                output_tail: String::new(),
16687                duration_ms: 0,
16688                resource_blocked: false,
16689            }],
16690            verify_retried: false,
16691            e2e_deferred: false,
16692            e2e_defer_reason: None,
16693            fix: Some(FixRecord {
16694                agent: "alpha".to_owned(),
16695                addressed: Vec::new(),
16696                rejected: vec![crate::verdict::Rejection {
16697                    id: "R1-1-1".to_owned(),
16698                    why: "not reachable from any caller".to_owned(),
16699                }],
16700                notes: String::new(),
16701                committed: true,
16702                failed: None,
16703                duration_ms: 0,
16704                continuation: None,
16705            }),
16706            blocking: 0,
16707            answered: 1,
16708            expected: 1,
16709            clean: false,
16710            progressed: true,
16711            vote_split: false,
16712            reconsideration: Vec::new(),
16713            verdict: None,
16714        };
16715        let state = state_with_round(round);
16716        let body = pr_message(&state, 'A').body;
16717
16718        assert!(body.contains("add retries"), "the task must still be there");
16719        assert!(body.contains("R2-1-1"), "{body}");
16720        assert!(!body.contains("unused import"), "{body}");
16721        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16722        assert!(
16723            !body.contains("not reachable from any caller"),
16724            "the reason it was declined: {body}"
16725        );
16726    }
16727
16728    #[test]
16729    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16730        let round = ReviewRound {
16731            round: 1,
16732            head: "deadbee".to_owned(),
16733            verified_head: None,
16734            verified_at: None,
16735            reviews: vec![ReviewRecord {
16736                attempts: 0,
16737                reviewer: 1,
16738                agent: "alpha".to_owned(),
16739                summary: String::new(),
16740                findings: Vec::new(),
16741                vote: None,
16742                failed: None,
16743                duration_ms: 0,
16744            }],
16745            e2e: Vec::new(),
16746            verify_retried: false,
16747            e2e_deferred: false,
16748            e2e_defer_reason: None,
16749            fix: None,
16750            blocking: 0,
16751            answered: 1,
16752            expected: 1,
16753            clean: true,
16754            progressed: false,
16755            vote_split: false,
16756            reconsideration: Vec::new(),
16757            verdict: None,
16758        };
16759        let state = state_with_round(round);
16760        let body = pr_message(&state, 'A').body;
16761        assert!(!body.contains("Open review findings"), "{body}");
16762        assert!(!body.contains("Declined"), "{body}");
16763    }
16764
16765    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16766        let mut state = RunState::new(
16767            PathBuf::from("/repo"),
16768            "main".to_owned(),
16769            "abc1234".to_owned(),
16770            instruction.to_owned(),
16771            Config::default(),
16772        );
16773        state.candidates.push(Candidate {
16774            index: 0,
16775            label: 'A',
16776            agent: "alpha".to_owned(),
16777            branch: "magi/x/A".to_owned(),
16778            worktree: PathBuf::from("/wt"),
16779            summary: summary.to_owned(),
16780            stat: String::new(),
16781            files: 1,
16782            commits: 1,
16783            empty: false,
16784            failed: None,
16785            verified_noop: None,
16786            folded: false,
16787            duration_ms: 0,
16788        });
16789        state
16790    }
16791
16792    fn review_state(subjects: &[&str]) -> RunState {
16793        let mut state = state_with_summary(
16794            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16795            "",
16796        );
16797        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16798        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16799        state
16800    }
16801
16802    /// A branch rebased onto a main that moved past the recorded
16803    /// `base_commit` is titled from its own first commit, never main's.
16804    #[tokio::test]
16805    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16806        ask_test_home();
16807        let tmp = tempfile::tempdir().unwrap();
16808        let repo = tmp.path().join("repo");
16809        std::fs::create_dir_all(&repo).unwrap();
16810        init_repo(&repo);
16811        let origin = tmp.path().join("origin.git");
16812        let g = |dir: &Path, args: &[&str]| {
16813            let out = std::process::Command::new("git")
16814                .args(args)
16815                .current_dir(dir)
16816                .quiet()
16817                .output()
16818                .expect("spawn git");
16819            assert!(
16820                out.status.success(),
16821                "git {args:?}: {}",
16822                String::from_utf8_lossy(&out.stderr)
16823            );
16824        };
16825        g(
16826            tmp.path(),
16827            &[
16828                "clone",
16829                "--bare",
16830                "-q",
16831                repo.to_str().unwrap(),
16832                origin.to_str().unwrap(),
16833            ],
16834        );
16835        g(
16836            &repo,
16837            &["remote", "add", "origin", origin.to_str().unwrap()],
16838        );
16839        let c1 = git::rev_parse(&repo, "main").await.unwrap();
16840
16841        // Main moves on; the branch is built on top of the new main.
16842        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16843        g(&repo, &["add", "-A"]);
16844        g(
16845            &repo,
16846            &["commit", "-q", "-m", "chore(deps): update a crate"],
16847        );
16848        g(&repo, &["push", "-q", "origin", "main"]);
16849        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16850        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16851        g(&repo, &["add", "-A"]);
16852        g(
16853            &repo,
16854            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16855        );
16856        g(&repo, &["checkout", "-q", "main"]);
16857
16858        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16859        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16860        // Without a readable tracking ref the recorded base's merge base is used.
16861        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16862        assert_eq!(fallback, c1);
16863
16864        let mut state = review_state(&[
16865            "chore(deps): update a crate",
16866            "fix(daemon): apply a chosen action",
16867        ]);
16868        state.repo = repo.clone();
16869        state.base_branch = "main".to_owned();
16870        state.base_commit = c1;
16871        refresh_reviewed_commits(&mut state, "feat/own").await;
16872        assert_eq!(
16873            state.reviewed_commits,
16874            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16875        );
16876        assert_eq!(
16877            review_title(&state).as_deref(),
16878            Some("fix(daemon): apply a chosen action")
16879        );
16880        assert_eq!(
16881            leaked_subjects(&state, "feat/own").await,
16882            Some(vec!["chore(deps): update a crate".to_owned()])
16883        );
16884        // A stale tracking ref is still used when the fetch fails, but the
16885        // leak list is withheld.
16886        state.config.merge.remote = "nowhere".to_owned();
16887        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16888    }
16889
16890    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
16891    /// with `origin/main` one commit behind it, and a summarizing agent that
16892    /// prints whatever `reply` holds.
16893    struct SummaryFixture {
16894        _tmp: tempfile::TempDir,
16895        repo: PathBuf,
16896        reply: PathBuf,
16897        runner: Runner,
16898    }
16899
16900    fn summary_fixture() -> SummaryFixture {
16901        ask_test_home();
16902        let tmp = tempfile::tempdir().unwrap();
16903        let repo = tmp.path().join("repo");
16904        std::fs::create_dir_all(&repo).unwrap();
16905        init_repo(&repo);
16906        let origin = tmp.path().join("origin.git");
16907        let g = |args: &[&str]| {
16908            let out = std::process::Command::new("git")
16909                .args(args)
16910                .current_dir(&repo)
16911                .quiet()
16912                .output()
16913                .expect("spawn git");
16914            assert!(
16915                out.status.success(),
16916                "git {args:?}: {}",
16917                String::from_utf8_lossy(&out.stderr)
16918            );
16919        };
16920        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16921        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16922        g(&["push", "-q", "origin", "main"]);
16923        g(&["checkout", "-q", "-b", "feat/own"]);
16924        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16925        g(&["add", "-A"]);
16926        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16927        let reply = tmp.path().join("reply.txt");
16928        std::fs::write(
16929            &reply,
16930            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16931        )
16932        .unwrap();
16933        let mut runner = runner_at(RunStatus::Gating);
16934        let mut state = review_state(&["feat(own): add the own file"]);
16935        state.repo = repo.clone();
16936        state.base_branch = "main".to_owned();
16937        state.candidates[0].branch = "feat/own".to_owned();
16938        state.candidates[0].worktree = repo.clone();
16939        let mut author = spec("alpha");
16940        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16941        author
16942            .env
16943            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16944        state.config.agents = vec![author];
16945        runner.state = state;
16946        SummaryFixture {
16947            _tmp: tmp,
16948            repo,
16949            reply,
16950            runner,
16951        }
16952    }
16953
16954    async fn posted(f: &mut SummaryFixture) -> PrMessage {
16955        let winner = f.runner.state.candidates[0].clone();
16956        f.runner
16957            .guarded_pr_message(&winner, None, true)
16958            .await
16959            .unwrap()
16960            .expect("an English description never parks")
16961    }
16962
16963    fn clean_round_with_minor_findings() -> ReviewRound {
16964        let mut round = passing_round(2, None);
16965        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16966        round.reviews = vec![ReviewRecord {
16967            attempts: 0,
16968            reviewer: 1,
16969            agent: "alpha".into(),
16970            summary: "内部レビュー".into(),
16971            findings: vec![
16972                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16973                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16974            ],
16975            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16976            failed: None,
16977            duration_ms: 0,
16978        }];
16979        round
16980    }
16981
16982    #[tokio::test]
16983    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16984        for review_only in [false, true] {
16985            let mut f = summary_fixture();
16986            if !review_only {
16987                f.runner.state.candidates[0].agent = "alpha".into();
16988            }
16989            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16990            assert!(f.runner.state.reviews[0].clean);
16991            assert!(f.runner.state.reviews[0].fix.is_none());
16992            assert!(f.runner.state.open_findings().is_empty());
16993            assert_eq!(
16994                crate::pr_outcome::required_finding_ids(&f.runner.state),
16995                ["R2-1-1", "R2-1-2"]
16996            );
16997            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16998            let evidence = crate::pr_outcome::summary_evidence(
16999                &f.runner.state,
17000                crate::pr_outcome::FinalHead::At(&head),
17001            );
17002            assert!(
17003                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17004            );
17005            // The ordinary reply omits both IDs: no cached description is adopted.
17006            let omitted = posted(&mut f).await;
17007            assert!(omitted.body.contains("No description"));
17008            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17009            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17010            assert!(f.runner.state.change_summary.is_none());
17011            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17012            let described = posted(&mut f).await;
17013            assert!(described.body.contains("improve documentation for callers"));
17014            assert!(described.body.contains("remaining edge case"));
17015            assert!(described.body.contains("## Open review findings"));
17016            assert!(!described.body.contains("未修正"));
17017        }
17018    }
17019
17020    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17021        let mut f = summary_fixture();
17022        if !review_only {
17023            f.runner.state.candidates[0].agent = "alpha".into();
17024        }
17025        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17026        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17027        fixed
17028            .fix
17029            .as_mut()
17030            .unwrap()
17031            .rejected
17032            .push(crate::verdict::Rejection {
17033                id: "R1-1-1".into(),
17034                why: "呼び出されないコードのため辞退".into(),
17035            });
17036        let mut clean = clean_round_with_minor_findings();
17037        clean.verified_head = Some(head.clone());
17038        f.runner.state.reviews = vec![fixed, clean];
17039        ok_gate(&mut f.runner.state, Some(&head));
17040        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17041        f.runner.state.config.agents.push(scripted("beta", reply));
17042        f.runner.state.config.roles.language_judge =
17043            Some(crate::config::AgentChoice::One("beta".into()));
17044        f
17045    }
17046
17047    #[tokio::test]
17048    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17049        for review_only in [false, true] {
17050            let mut f = rewrite_fixture_with_findings(review_only,
17051                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17052            let m = posted(&mut f).await;
17053            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17054            let expected =
17055                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17056            assert!(m.body.contains(&expected));
17057            assert!(m.body.contains("## Review rounds"));
17058            assert!(
17059                m.body
17060                    .contains("Gate (`verify.gate`): passed on the final commit")
17061            );
17062            assert!(
17063                m.body
17064                    .contains("Tests (`verify.e2e`): passed on the final commit")
17065            );
17066            let c = f.runner.state.change_summary.as_ref().unwrap();
17067            assert!(
17068                m.body
17069                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17070            );
17071            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17072            assert!(crate::pr_outcome::covers_required_findings(
17073                &f.runner.state,
17074                &m.body
17075            ));
17076            assert!(m.body.contains("documentation remains incomplete"));
17077            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17078            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17079            assert!(f.runner.state.github_text.is_none());
17080        }
17081    }
17082
17083    #[tokio::test]
17084    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17085        for review_only in [false, true] {
17086            for reply in [
17087                "TITLE: fix: public description\nA short English description.",
17088                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17089            ] {
17090                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17091                let winner = f.runner.state.candidates[0].clone();
17092                let posted = f
17093                    .runner
17094                    .guarded_pr_message(&winner, None, true)
17095                    .await
17096                    .unwrap();
17097                assert!(posted.is_none());
17098                assert!(f.runner.state.parked);
17099                let withheld = f.runner.state.github_text.as_ref().unwrap();
17100                assert!(
17101                    withheld.rewrite.is_none(),
17102                    "an incomplete rewrite must not be selectable"
17103                );
17104                let q = ask::Questions::open()
17105                    .get(withheld.question.as_ref().unwrap())
17106                    .unwrap();
17107                assert!(
17108                    !q.choices
17109                        .contains(&crate::github_text::USE_REWRITE.to_owned())
17110                );
17111                assert!(
17112                    q.detail.contains("R1-1-1")
17113                        && q.detail.contains("R2-1-1")
17114                        && q.detail.contains("R2-1-2")
17115                );
17116            }
17117        }
17118    }
17119
17120    #[tokio::test]
17121    async fn japanese_reports_stay_local_in_both_public_run_modes() {
17122        for review_only in [false, true] {
17123            let mut f = summary_fixture();
17124            if !review_only {
17125                f.runner.state.candidates[0].agent = "alpha".into();
17126            }
17127            f.runner.state.instruction = "内部タスクの説明".into();
17128            f.runner.state.candidates[0].summary = "内部の候補報告".into();
17129            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17130            f.runner.state.reviews = vec![
17131                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17132                round_with_notes(2, Some("二回目の内部修正報告です。")),
17133                passing_round(3, Some(&head)),
17134            ];
17135            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17136            ok_gate(&mut f.runner.state, Some(&head));
17137            let captured = f.repo.join("public-prompt.txt");
17138            let author = &mut f.runner.state.config.agents[0];
17139            author.command = vec![
17140                "sh".into(),
17141                "-c".into(),
17142                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17143            ];
17144            author.env.insert(
17145                "PROMPT_FILE".into(),
17146                captured.to_string_lossy().into_owned(),
17147            );
17148            let m = posted(&mut f).await;
17149            assert!(m.body.contains("Adds `own.txt`"));
17150            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17151            assert!(m.body.contains("R1-1-1"));
17152            assert!(
17153                m.body
17154                    .contains("Gate (`verify.gate`): passed on the final commit")
17155            );
17156            assert!(
17157                m.body
17158                    .contains("Tests (`verify.e2e`): passed on the final commit")
17159            );
17160            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17161            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17162            let prompt = std::fs::read_to_string(captured).unwrap();
17163            assert!(prompt.contains(&format!(
17164                "git diff {} {head}",
17165                f.runner.state.change_summary.as_ref().unwrap().base
17166            )));
17167            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17168            assert_eq!(
17169                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17170                "二回目の内部修正報告です。"
17171            );
17172        }
17173    }
17174
17175    #[tokio::test]
17176    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17177        for review_only in [false, true] {
17178            let mut f = summary_fixture();
17179            if !review_only {
17180                f.runner.state.candidates[0].agent = "alpha".into();
17181            }
17182            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17183            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17184            std::fs::write(&f.reply, "no usable description").unwrap();
17185            let failed = posted(&mut f).await;
17186            assert!(
17187                failed
17188                    .body
17189                    .contains("No description of what this change does could be generated")
17190            );
17191            assert!(failed.body.is_ascii());
17192            assert!(f.runner.state.github_text.is_none());
17193            assert!(f.runner.state.change_summary.is_none());
17194            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17195            let evidence = crate::pr_outcome::summary_evidence(
17196                &f.runner.state,
17197                crate::pr_outcome::FinalHead::At(&head),
17198            );
17199            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17200            assert!(
17201                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17202                    .exists()
17203            );
17204            f.runner.state =
17205                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17206            std::fs::write(
17207                &f.reply,
17208                "TITLE: feat(own): add the own file\nAdds the committed own file.",
17209            )
17210            .unwrap();
17211            let retried = posted(&mut f).await;
17212            assert!(retried.body.contains("Adds the committed own file."));
17213            assert_eq!(
17214                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17215                "日本語の修正報告"
17216            );
17217        }
17218    }
17219
17220    #[tokio::test]
17221    async fn same_head_verification_and_findings_invalidate_public_prose() {
17222        let mut f = summary_fixture();
17223        posted(&mut f).await;
17224        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17225        let original = f
17226            .runner
17227            .state
17228            .change_summary
17229            .as_ref()
17230            .unwrap()
17231            .evidence
17232            .clone();
17233        ok_gate(&mut f.runner.state, Some(&head));
17234        std::fs::write(
17235            &f.reply,
17236            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17237        )
17238        .unwrap();
17239        let verified = posted(&mut f).await;
17240        assert!(verified.body.contains("now verified"));
17241        assert_ne!(
17242            f.runner.state.change_summary.as_ref().unwrap().evidence,
17243            original
17244        );
17245        let mut round = round_with_notes(1, Some("内部報告"));
17246        round.reviews.push(ReviewRecord {
17247            attempts: 0,
17248            reviewer: 1,
17249            agent: "alpha".into(),
17250            summary: "内部レビュー".into(),
17251            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17252            vote: None,
17253            failed: None,
17254            duration_ms: 0,
17255        });
17256        round
17257            .fix
17258            .as_mut()
17259            .unwrap()
17260            .rejected
17261            .push(crate::verdict::Rejection {
17262                id: "R1-1-1".into(),
17263                why: "呼び出し元から到達できない".into(),
17264            });
17265        f.runner.state.reviews = vec![round];
17266        // A description omitting the required ID is unusable, even with a title.
17267        let omitted = posted(&mut f).await;
17268        assert!(omitted.body.contains("No description"));
17269        assert!(omitted.body.contains("R1-1-1"));
17270        assert!(!omitted.body.contains("到達"));
17271        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17272        let described = posted(&mut f).await;
17273        assert!(described.body.contains("unreachable from callers"));
17274        assert!(described.body.contains("R1-1-1"));
17275        assert!(!described.body.contains("到達"));
17276        let evidence = f
17277            .runner
17278            .state
17279            .change_summary
17280            .as_ref()
17281            .unwrap()
17282            .evidence
17283            .clone();
17284        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17285        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17286        assert!(posted(&mut f).await.body.contains("rationale updated"));
17287        assert_ne!(
17288            f.runner.state.change_summary.as_ref().unwrap().evidence,
17289            evidence
17290        );
17291        // Old serialized summaries have no evidence fingerprint and must regenerate.
17292        f.runner
17293            .state
17294            .change_summary
17295            .as_mut()
17296            .unwrap()
17297            .evidence
17298            .clear();
17299        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17300        assert!(posted(&mut f).await.body.contains("current specification"));
17301    }
17302
17303    #[tokio::test]
17304    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17305        let mut f = summary_fixture();
17306        let m = posted(&mut f).await;
17307        assert_eq!(m.title, "feat(own): add the own file");
17308        assert!(
17309            m.body
17310                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17311            "{}",
17312            m.body
17313        );
17314        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17315        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17316        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17317        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17318        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17319        let kept = f.runner.state.change_summary.as_ref().unwrap();
17320        assert_eq!(
17321            (kept.head.as_str(), kept.base.as_str()),
17322            (head.as_str(), base.as_str())
17323        );
17324        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17325    }
17326
17327    #[tokio::test]
17328    async fn review_only_summary_receives_the_configured_github_language() {
17329        let mut f = summary_fixture();
17330        f.runner.state.config.graph.github_language = "ja".into();
17331        let captured = f.repo.join("summary-prompt.txt");
17332        let author = &mut f.runner.state.config.agents[0];
17333        author.command = vec![
17334            "sh".into(),
17335            "-c".into(),
17336            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17337        ];
17338        author.env.insert(
17339            "PROMPT_FILE".into(),
17340            captured.to_string_lossy().into_owned(),
17341        );
17342        std::fs::write(
17343            &f.reply,
17344            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17345        )
17346        .unwrap();
17347        let m = posted(&mut f).await;
17348        let prompt = std::fs::read_to_string(captured).unwrap();
17349        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17350        assert!(!prompt.contains("in English"));
17351        assert_eq!(m.title, "feat(own): ファイルを追加");
17352        assert!(m.body.contains("own.txt を追加します。"));
17353    }
17354
17355    #[tokio::test]
17356    async fn review_fixes_come_after_the_whole_change_summary() {
17357        let mut f = summary_fixture();
17358        f.runner.state.reviews = vec![
17359            round_with_notes(1, Some("handled the PATH case")),
17360            round_with_notes(2, None),
17361        ];
17362        let m = posted(&mut f).await;
17363        let summary = m.body.find("## Summary").expect("summary");
17364        let fixes = m.body.find("## Review rounds").expect("review rounds");
17365        assert!(summary < fixes, "{}", m.body);
17366        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17367        assert!(m.body.contains("## Final verification"), "{}", m.body);
17368    }
17369
17370    #[tokio::test]
17371    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17372        let mut f = summary_fixture();
17373        let first = posted(&mut f).await;
17374        let first_head = f.runner.state.change_summary.clone().unwrap().head;
17375        // The same head reuses the record: a different reply is not read.
17376        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17377        let again = posted(&mut f).await;
17378        assert_eq!((again.title, again.body), (first.title, first.body));
17379
17380        let g = |args: &[&str]| {
17381            let out = std::process::Command::new("git")
17382                .args(args)
17383                .current_dir(&f.repo)
17384                .quiet()
17385                .output()
17386                .expect("spawn git");
17387            assert!(out.status.success(), "git {args:?}");
17388        };
17389        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17390        g(&["add", "-A"]);
17391        g(&["commit", "-q", "-m", "fix(own): handle more"]);
17392        let second = posted(&mut f).await;
17393        let kept = f.runner.state.change_summary.clone().unwrap();
17394        assert_ne!(kept.head, first_head);
17395        assert!(second.body.contains("Not read yet."), "{}", second.body);
17396        assert!(!second.body.contains("so the first summary is read"));
17397        assert_eq!(second.title, "feat(own): other");
17398    }
17399
17400    #[tokio::test]
17401    async fn an_unavailable_summary_is_stated_and_never_invented() {
17402        let mut f = summary_fixture();
17403        std::fs::write(&f.reply, "no title line here").unwrap();
17404        let m = posted(&mut f).await;
17405        assert!(
17406            m.body
17407                .contains("No description of what this change does could be generated"),
17408            "{}",
17409            m.body
17410        );
17411        assert!(m.body.contains("internal reports remain in local records"));
17412        assert!(f.runner.state.change_summary.is_none());
17413        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17414        // A preview asks nobody and claims nothing.
17415        let winner = f.runner.state.candidates[0].clone();
17416        let preview = f
17417            .runner
17418            .guarded_pr_message(&winner, None, false)
17419            .await
17420            .unwrap()
17421            .unwrap();
17422        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17423    }
17424
17425    #[test]
17426    fn pr_message_review_single_commit_uses_its_subject() {
17427        let state = review_state(&["feat(nats): per-role user"]);
17428        let m = pr_message(&state, 'A');
17429        assert_eq!(m.title, "feat(nats): per-role user");
17430        assert!(!m.body.contains("Review the work already"), "{}", m.body);
17431        assert!(m.body.contains("## Commits under review"), "{}", m.body);
17432    }
17433
17434    #[test]
17435    fn pr_message_review_multi_commit_takes_the_oldest() {
17436        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17437        let m = pr_message(&state, 'A');
17438        assert_eq!(m.title, "feat: the change");
17439        for s in ["feat: the change", "fix: typo", "fix: again"] {
17440            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17441        }
17442    }
17443
17444    #[test]
17445    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17446        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17447            let mut state = review_state(&[first, "fix: later fixup"]);
17448            state.candidates[0].branch = "機能/ログイン".to_owned();
17449            let m = pr_message(&state, 'A');
17450            assert!(
17451                m.title.starts_with("chore: land candidate A of run"),
17452                "{}",
17453                m.title
17454            );
17455        }
17456    }
17457
17458    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17459        BranchFacts {
17460            commits: commits
17461                .iter()
17462                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17463                .collect(),
17464            stat: stat.to_owned(),
17465        }
17466    }
17467
17468    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17469        let mut r = review_round(true, 0, 1, 1, true, true);
17470        r.round = round;
17471        r.fix = notes.map(|n| FixRecord {
17472            agent: "fixer".to_owned(),
17473            addressed: Vec::new(),
17474            rejected: Vec::new(),
17475            notes: n.to_owned(),
17476            committed: true,
17477            failed: None,
17478            duration_ms: 0,
17479            continuation: None,
17480        });
17481        r
17482    }
17483
17484    #[test]
17485    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17486        let state = review_state(&["ignored"]);
17487        let f = facts(
17488            &[
17489                (
17490                    "fix(login): resolve PATH on macOS",
17491                    "Login shells skip rc files.",
17492                ),
17493                ("fix: address review", ""),
17494            ],
17495            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17496        );
17497        let m = pr_message_with(&state, 'A', Some(&f));
17498        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17499        assert!(
17500            m.body
17501                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
17502            "{}",
17503            m.body
17504        );
17505        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17506        assert!(m.body.contains("## Diff stat"), "{}", m.body);
17507        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17508        for banned in [
17509            "Review the work already",
17510            "no task statement",
17511            "Original task",
17512        ] {
17513            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17514        }
17515        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17516    }
17517
17518    #[test]
17519    fn pr_message_review_truncates_a_huge_first_commit_body() {
17520        let state = review_state(&["ignored"]);
17521        let f = facts(
17522            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17523            "s",
17524        );
17525        let m = pr_message_with(&state, 'A', Some(&f));
17526        assert!(m.body.len() < 30_000, "{}", m.body.len());
17527        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17528        assert!(
17529            m.body.contains("- ... 1 more commit(s)"),
17530            "{}",
17531            m.body.len()
17532        );
17533        assert!(m.body.ends_with("magi:candidate-a\n"));
17534    }
17535
17536    #[test]
17537    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17538        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17539        assert_eq!(m.title, "feat: x");
17540        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17541        assert!(!m.body.contains("Diff stat"), "{}", m.body);
17542        assert!(!m.body.contains("no task statement"), "{}", m.body);
17543    }
17544
17545    #[test]
17546    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17547        let mut state = review_state(&["日本語の件名"]);
17548        state.candidates[0].branch = "fix/macos-login-path".to_owned();
17549        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17550        state.candidates[0].branch = "機能/ログイン".to_owned();
17551        assert!(
17552            pr_message(&state, 'A')
17553                .title
17554                .starts_with("chore: land candidate A")
17555        );
17556    }
17557
17558    #[test]
17559    fn pr_message_review_fixes_survive_a_clean_final_round() {
17560        let mut state = review_state(&["feat: x"]);
17561        state.reviews = vec![
17562            round_with_notes(1, Some("handled the PATH case")),
17563            round_with_notes(2, None),
17564        ];
17565        let body = pr_message(&state, 'A').body;
17566        assert!(
17567            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17568            "{body}"
17569        );
17570        assert!(body.contains("### Round 1"), "{body}");
17571
17572        state.reviews = vec![
17573            round_with_notes(1, Some("first fix")),
17574            round_with_notes(2, Some("")),
17575            round_with_notes(3, Some("second fix")),
17576            round_with_notes(4, None),
17577        ];
17578        let body = pr_message(&state, 'A').body;
17579        assert!(
17580            body.contains("### Round 1") && !body.contains("first fix"),
17581            "{body}"
17582        );
17583        assert!(
17584            body.contains("### Round 3") && !body.contains("second fix"),
17585            "{body}"
17586        );
17587        assert!(body.contains("### Round 2"), "{body}");
17588    }
17589
17590    const FINAL: &str = "abcdef1234567890";
17591
17592    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17593        let mut r = round_with_notes(round, None);
17594        r.verified_head = verified.map(str::to_owned);
17595        r
17596    }
17597
17598    fn ok_gate(state: &mut RunState, head: Option<&str>) {
17599        state.gate = vec![CommandOutcome {
17600            command: "gate".to_owned(),
17601            code: Some(0),
17602            output_tail: String::new(),
17603            duration_ms: 0,
17604            resource_blocked: false,
17605        }];
17606        state.gate_ran = true;
17607        state.gate_head = head.map(str::to_owned);
17608    }
17609
17610    fn body_at(state: &RunState) -> String {
17611        pr_message_raw(
17612            state,
17613            'A',
17614            None,
17615            crate::pr_outcome::FinalHead::At(FINAL),
17616            ChangeNote::NotAsked,
17617        )
17618        .body
17619    }
17620
17621    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17622
17623    #[test]
17624    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17625        let mut state = review_state(&["feat: x"]);
17626        state.reviews = vec![
17627            round_with_notes(1, Some(SANDBOXED)),
17628            passing_round(2, Some(FINAL)),
17629        ];
17630        ok_gate(&mut state, Some(FINAL));
17631        let body = body_at(&state);
17632        assert!(!body.contains("Handled the PATH case."), "{body}");
17633        assert!(!body.contains("could not commit"), "{body}");
17634        assert!(!body.contains("could not run"), "{body}");
17635        assert!(
17636            body.contains("The fix is committed on the branch"),
17637            "{body}"
17638        );
17639        assert!(!body.contains("sandbox"), "{body}");
17640        assert!(
17641            body.contains("Tests (`verify.e2e`): passed on the final commit"),
17642            "{body}"
17643        );
17644        assert!(
17645            body.contains("Gate (`verify.gate`): passed on the final commit"),
17646            "{body}"
17647        );
17648        // The raw report is untouched in the record.
17649        assert!(
17650            state.reviews[0]
17651                .fix
17652                .as_ref()
17653                .unwrap()
17654                .notes
17655                .contains("could not commit")
17656        );
17657    }
17658
17659    #[test]
17660    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17661        let mut state = review_state(&["feat: x"]);
17662        let mut r1 = round_with_notes(1, Some(SANDBOXED));
17663        r1.fix.as_mut().unwrap().committed = false;
17664        let mut r2 = passing_round(2, Some(FINAL));
17665        r2.e2e[0].code = Some(1);
17666        state.reviews = vec![r1, r2];
17667        let body = body_at(&state);
17668        assert!(!body.contains("could not commit"), "{body}");
17669        assert!(!body.contains("could not run"), "{body}");
17670        assert!(body.contains("This round produced no commit"), "{body}");
17671        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17672        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17673        assert!(!body.contains("passed"), "{body}");
17674    }
17675
17676    #[test]
17677    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17678        let mut state = review_state(&["feat: x"]);
17679        state.reviews = vec![
17680            round_with_notes(1, Some(SANDBOXED)),
17681            passing_round(2, Some("0123456789abcdef")),
17682        ];
17683        ok_gate(&mut state, Some("0123456789abcdef"));
17684        let body = body_at(&state);
17685        assert!(!body.contains("HTTP tests could not run"), "{body}");
17686        // The commit claim is still settled by the structured record.
17687        assert!(!body.contains("could not commit"), "{body}");
17688        assert!(
17689            body.contains("passed on an earlier commit (`0123456`)"),
17690            "{body}"
17691        );
17692        assert!(!body.contains("passed on the final commit"), "{body}");
17693
17694        // A gate recorded before `gate_head` existed is "not recorded".
17695        ok_gate(&mut state, None);
17696        let body = body_at(&state);
17697        assert!(
17698            body.contains("passed on a commit that was not recorded"),
17699            "{body}"
17700        );
17701    }
17702
17703    #[test]
17704    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17705        let state = state_with_summary(
17706            "add retries to the client",
17707            "TITLE: feat: retries\n\nDid it.",
17708        );
17709        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17710        assert_eq!(m.title, "feat: retries");
17711        assert!(
17712            m.body.contains("<summary>Original task</summary>"),
17713            "{}",
17714            m.body
17715        );
17716        assert!(!m.body.contains("Commits under review"), "{}", m.body);
17717        assert!(
17718            m.body
17719                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17720            "{}",
17721            m.body
17722        );
17723    }
17724
17725    #[test]
17726    fn pr_message_review_bounds_a_long_english_subject() {
17727        let long = format!("feat: {}", "word ".repeat(100));
17728        let m = pr_message(&review_state(&[&long]), 'A');
17729        assert!(m.title.starts_with("feat: word"), "{}", m.title);
17730        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17731    }
17732
17733    #[test]
17734    fn pr_message_implementation_run_is_unchanged_by_review_support() {
17735        let state = state_with_summary("add retries\n\ndetails", "- did some things");
17736        let m = pr_message(&state, 'A');
17737        assert_eq!(m.title, "add retries");
17738        assert!(m.body.contains("<summary>Original task</summary>"));
17739        assert!(!m.body.contains("Commits under review"));
17740        assert_eq!(landing_subject_source(&state), state.instruction);
17741    }
17742
17743    #[test]
17744    fn review_run_squash_subject_is_the_change_not_the_prompt() {
17745        let state = review_state(&["feat: the change", "fix: typo"]);
17746        let source = landing_subject_source(&state);
17747        assert_eq!(land::merge_subject("", &source), "feat: the change");
17748        assert_eq!(
17749            land::merge_subject("magi: candidate A (uncommitted work)", &source),
17750            "feat: the change"
17751        );
17752        // An operator's rename still wins.
17753        assert_eq!(
17754            land::merge_subject("feat: renamed by hand", &source),
17755            "feat: renamed by hand"
17756        );
17757        let blank = review_state(&["日本語"]);
17758        assert!(
17759            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17760        );
17761    }
17762
17763    #[test]
17764    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17765        let state = review_state(&["feat: the change"]);
17766        let source = landing_subject_source(&state);
17767        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17768        assert_eq!(
17769            land::merge_subject(landing_title(&state, old), &source),
17770            "feat: the change"
17771        );
17772        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17773        let task = state_with_summary("add retries", "");
17774        assert_eq!(landing_title(&task, old), old);
17775    }
17776
17777    #[test]
17778    fn pr_message_describes_the_change_not_the_task() {
17779        let state = state_with_summary(
17780            "今回やってほしいこと: results projector を直す",
17781            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17782        );
17783        let m = pr_message(&state, 'A');
17784        assert_eq!(m.title, "fix(web): batch the runs list reads");
17785        assert!(
17786            m.body.starts_with("## Summary\n\n- reads run.json once"),
17787            "{}",
17788            m.body
17789        );
17790        assert!(!m.body.contains("TITLE:"), "{}", m.body);
17791        let task_at = m.body.find("今回やってほしいこと").unwrap();
17792        let details_at = m.body.find("<details>").unwrap();
17793        assert!(
17794            details_at < task_at,
17795            "the task lives inside <details>: {}",
17796            m.body
17797        );
17798        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17799        assert!(m.body.contains("magi:candidate-a"));
17800    }
17801
17802    #[test]
17803    fn pr_message_falls_back_to_the_task_without_a_title_line() {
17804        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17805        let m = pr_message(&state, 'A');
17806        assert_eq!(m.title, "add retries");
17807        assert!(
17808            m.body.contains("## Summary\n\n- did some things"),
17809            "{}",
17810            m.body
17811        );
17812
17813        let none = RunState::new(
17814            PathBuf::from("/repo"),
17815            "main".to_owned(),
17816            "abc1234".to_owned(),
17817            "add retries".to_owned(),
17818            Config::default(),
17819        );
17820        let m = pr_message(&none, 'A');
17821        assert_eq!(m.title, "add retries");
17822        assert!(!m.body.contains("## Summary"), "{}", m.body);
17823    }
17824
17825    #[test]
17826    fn pr_message_refuses_the_candidate_commit_subject() {
17827        for bad in [
17828            "TITLE: magi: candidate A (uncommitted work)",
17829            "TITLE: chore: stuff (uncommitted work)",
17830            "TITLE:   ",
17831        ] {
17832            let state = state_with_summary("add retries", bad);
17833            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17834        }
17835    }
17836
17837    #[test]
17838    fn pr_message_bounds_a_very_long_task_and_title() {
17839        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17840        let state = state_with_summary(&long, "- nothing");
17841        let m = pr_message(&state, 'A');
17842        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17843        assert!(!m.title.contains('\n'));
17844
17845        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17846        let m = pr_message(&state, 'A');
17847        assert!(m.title.starts_with("feat: "));
17848        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17849        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17850    }
17851
17852    fn long_title_of(instruction: &str) -> String {
17853        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17854    }
17855
17856    #[test]
17857    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17858        let first = "Make the landing path keep a readable title for long tasks";
17859        let line = format!(
17860            "{first}. {}",
17861            "And then keep going with more words ".repeat(20)
17862        );
17863        let t = long_title_of(&line);
17864        assert_eq!(t, first);
17865        assert!(!t.starts_with("chore: land"));
17866    }
17867
17868    #[test]
17869    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17870        let line = "word ".repeat(200);
17871        let t = long_title_of(&line);
17872        assert!(t.ends_with("word..."), "{t}");
17873        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17874    }
17875
17876    #[test]
17877    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17878        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17879            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17880        }
17881    }
17882
17883    #[test]
17884    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17885        let mut state = state_with_summary("再試行を追加する", "- nothing");
17886        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17887        state.config.graph.github_language = "ja".into();
17888        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17889    }
17890
17891    #[test]
17892    fn pr_message_title_limit_is_exact() {
17893        let at = "a".repeat(PR_TITLE_MAX);
17894        assert_eq!(long_title_of(&at), at);
17895        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17896        assert!(over.ends_with("..."), "{over}");
17897        assert_eq!(over.chars().count(), PR_TITLE_MAX);
17898    }
17899
17900    #[test]
17901    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17902        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17903        let t = long_title_of(&line);
17904        assert!(t.ends_with("..."), "{t}");
17905        assert!(t.is_ascii(), "{t}");
17906    }
17907
17908    #[test]
17909    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17910        let line = format!(
17911            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17912            "plus more filler words ".repeat(20)
17913        );
17914        let t = long_title_of(&line);
17915        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17916    }
17917
17918    #[test]
17919    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17920        let line = format!("\n\n# {}", "title words ".repeat(40));
17921        let state = state_with_summary(&line, "- nothing");
17922        let m = pr_message(&state, 'A');
17923        assert!(m.title.starts_with("title words"), "{}", m.title);
17924        assert_eq!(
17925            land::merge_subject(&m.title, &landing_subject_source(&state)),
17926            m.title
17927        );
17928        // An operator's rename wins untouched.
17929        assert_eq!(
17930            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17931            "feat: renamed by hand"
17932        );
17933    }
17934
17935    #[tokio::test]
17936    async fn github_text_rewrite_is_bounded_and_falls_back() {
17937        crate::run::pin_test_home();
17938        for (reply, accepted) in [
17939            (
17940                "TITLE: fix: retries\nAdd retries for failed requests.",
17941                true,
17942            ),
17943            (
17944                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17945                false,
17946            ),
17947            (
17948                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17949                true,
17950            ),
17951            (
17952                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17953                true,
17954            ),
17955            (
17956                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17957                true,
17958            ),
17959            (
17960                "TITLE: fix: retries\n日本語の説明です。 token=secret",
17961                false,
17962            ),
17963            ("TITLE: fix: retries\n", false),
17964        ] {
17965            let dir = tempfile::tempdir().unwrap();
17966            let mut runner = runner_at(RunStatus::Gating);
17967            runner.state = state_with_summary(
17968                "add retries",
17969                "TITLE: fix: retries\n日本語の説明を書きます。",
17970            );
17971            runner.state.repo = dir.path().to_owned();
17972            runner.state.candidates[0].worktree = dir.path().to_owned();
17973            let mut author = spec("alpha");
17974            author.command = vec![
17975                "sh".into(),
17976                "-c".into(),
17977                "printf '%s' \"$REWRITE_REPLY\"".into(),
17978            ];
17979            author.env.insert("REWRITE_REPLY".into(), reply.into());
17980            runner.state.config.agents = vec![author];
17981            seed_public_summary(&mut runner);
17982            let winner = runner.state.candidates[0].clone();
17983            let first = runner
17984                .guarded_pr_message(&winner, None, true)
17985                .await
17986                .unwrap();
17987            let message = if accepted {
17988                first.expect("a passing rewrite never asks")
17989            } else {
17990                // Withheld: the run parks on a question and posts nothing yet.
17991                assert!(first.is_none());
17992                assert!(runner.state.parked);
17993                let store = ask::Questions::open();
17994                let asked: Vec<_> = store
17995                    .list()
17996                    .into_iter()
17997                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17998                    .collect();
17999                assert_eq!(asked.len(), 1);
18000                assert!(!asked[0].detail.contains("secret"));
18001                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18002                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18003                assert_eq!(
18004                    asked[0]
18005                        .choices
18006                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18007                    rewrite.is_some()
18008                );
18009                // Silence: the question lapses and the neutral text is used,
18010                // without a second rewrite call or a second question.
18011                store
18012                    .update(&asked[0].id, |q| {
18013                        q.abandon("test");
18014                        Ok(())
18015                    })
18016                    .unwrap();
18017                let again = runner
18018                    .guarded_pr_message(&winner, None, true)
18019                    .await
18020                    .unwrap()
18021                    .expect("an abandoned question falls back");
18022                let n = store
18023                    .list()
18024                    .into_iter()
18025                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18026                    .count();
18027                assert_eq!(n, 1, "asked once per rejected text");
18028                again
18029            };
18030            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18031            assert!(!message.body.contains("secret"));
18032            assert!(!message.body.contains("prefix"));
18033            assert!(!message.title.contains("secret"));
18034            assert_eq!(
18035                message.body.contains("Add retries for failed requests."),
18036                accepted
18037            );
18038            assert_eq!(
18039                runner.state.seats["github-text-rewrite"].turns, 1,
18040                "only one rewrite invocation"
18041            );
18042            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18043            if accepted {
18044                assert!(runner.state.github_text.is_none());
18045                assert!(!runner.state.parked);
18046                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18047                assert!(message.body.contains("## Final verification"));
18048                assert!(message.body.contains("Evidence: final diff"));
18049                assert!(
18050                    message
18051                        .body
18052                        .contains(&format!("magi:run/{}", runner.state.id))
18053                );
18054            }
18055            if !accepted {
18056                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18057                assert!(
18058                    message
18059                        .body
18060                        .contains(&format!("magi:run/{}", runner.state.id))
18061                );
18062            }
18063        }
18064    }
18065
18066    fn scripted(id: &str, reply: &str) -> AgentSpec {
18067        let mut a = spec(id);
18068        a.command = vec![
18069            "sh".into(),
18070            "-c".into(),
18071            "printf '%s' \"$REWRITE_REPLY\"".into(),
18072        ];
18073        a.env.insert("REWRITE_REPLY".into(), reply.into());
18074        a
18075    }
18076
18077    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18078    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18079
18080    /// Gate tests inject generated public output, not internal reports.
18081    fn seed_public_summary(runner: &mut Runner) {
18082        let repo = &runner.state.repo;
18083        let g = |args: &[&str]| {
18084            let out = std::process::Command::new("git")
18085                .args(args)
18086                .current_dir(repo)
18087                .quiet()
18088                .output()
18089                .unwrap();
18090            assert!(out.status.success(), "git {args:?}");
18091            String::from_utf8_lossy(&out.stdout).trim().to_owned()
18092        };
18093        g(&["init", "-q", "-b", "main"]);
18094        g(&[
18095            "-c",
18096            "user.name=Test",
18097            "-c",
18098            "user.email=test@example.invalid",
18099            "commit",
18100            "-q",
18101            "--allow-empty",
18102            "-m",
18103            "feat: fixture",
18104        ]);
18105        let head = g(&["rev-parse", "HEAD"]);
18106        g(&["update-ref", "refs/remotes/origin/main", &head]);
18107        let evidence = crate::pr_outcome::summary_evidence(
18108            &runner.state,
18109            crate::pr_outcome::FinalHead::At(&head),
18110        );
18111        let generated = &runner.state.candidates[0].summary;
18112        runner.state.change_summary = Some(crate::run::ChangeSummary {
18113            base: head.clone(),
18114            head,
18115            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18116            title: summary_title(generated).unwrap(),
18117            body: summary_without_title(generated),
18118        });
18119    }
18120
18121    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18122        let mut runner = runner_at(RunStatus::Gating);
18123        runner.state = state_with_summary("add retries", JA_SUMMARY);
18124        if review_only {
18125            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18126        }
18127        runner.state.repo = dir.to_owned();
18128        runner.state.candidates[0].worktree = dir.to_owned();
18129        runner.state.config.agents = agents;
18130        seed_public_summary(&mut runner);
18131        runner
18132    }
18133
18134    fn gate_questions(runner: &Runner) -> usize {
18135        ask::Questions::open()
18136            .list()
18137            .into_iter()
18138            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18139            .count()
18140    }
18141
18142    #[tokio::test]
18143    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18144        crate::run::pin_test_home();
18145        let dir = tempfile::tempdir().unwrap();
18146        let mut runner = rewrite_runner(
18147            dir.path(),
18148            true,
18149            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18150        );
18151        runner.state.config.roles.language_judge =
18152            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18153        let winner = runner.state.candidates[0].clone();
18154        let msg = runner
18155            .guarded_pr_message(&winner, None, true)
18156            .await
18157            .unwrap()
18158            .expect("an English rewrite posts without asking");
18159        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18160        assert!(msg.body.contains("<details>kept</details>"));
18161        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18162        assert_eq!(gate_questions(&runner), 0);
18163        assert!(runner.state.github_text.is_none());
18164        assert!(!runner.state.seats.contains_key("impl-A"));
18165        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18166    }
18167
18168    #[tokio::test]
18169    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18170        crate::run::pin_test_home();
18171        let dir = tempfile::tempdir().unwrap();
18172        let mut runner = rewrite_runner(
18173            dir.path(),
18174            false,
18175            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18176        );
18177        runner.state.config.roles.language_judge =
18178            Some(crate::config::AgentChoice::One("beta".into()));
18179        runner.state.seats.insert(
18180            "impl-A".into(),
18181            SeatState::new("impl-A", "alpha", runner.state.seed),
18182        );
18183        let winner = runner.state.candidates[0].clone();
18184        let msg = runner
18185            .guarded_pr_message(&winner, None, true)
18186            .await
18187            .unwrap()
18188            .expect("rewritten");
18189        assert!(msg.body.contains("<details>kept</details>"));
18190        assert_eq!(runner.state.seats["impl-A"].turns, 0);
18191        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18192        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18193    }
18194
18195    #[tokio::test]
18196    async fn rewrite_reuses_a_resumable_implementer_session() {
18197        crate::run::pin_test_home();
18198        let dir = tempfile::tempdir().unwrap();
18199        let mut runner = rewrite_runner(
18200            dir.path(),
18201            false,
18202            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18203        );
18204        runner.state.config.roles.language_judge =
18205            Some(crate::config::AgentChoice::One("beta".into()));
18206        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18207        seat.turns = 1;
18208        runner.state.seats.insert("impl-A".into(), seat);
18209        let winner = runner.state.candidates[0].clone();
18210        runner
18211            .guarded_pr_message(&winner, None, true)
18212            .await
18213            .unwrap()
18214            .expect("rewritten");
18215        assert_eq!(runner.state.seats["impl-A"].turns, 2);
18216        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18217    }
18218
18219    #[tokio::test]
18220    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18221        crate::run::pin_test_home();
18222        let dir = tempfile::tempdir().unwrap();
18223        let mut broken = spec("bad");
18224        broken.command = vec!["false".into()];
18225        let mut runner = rewrite_runner(
18226            dir.path(),
18227            true,
18228            vec![
18229                broken,
18230                scripted("noformat", "Add retries."),
18231                scripted("good", EN_REWRITE),
18232            ],
18233        );
18234        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18235            "bad".into(),
18236            "noformat".into(),
18237            "good".into(),
18238        ]));
18239        let winner = runner.state.candidates[0].clone();
18240        let msg = runner
18241            .guarded_pr_message(&winner, None, true)
18242            .await
18243            .unwrap()
18244            .expect("the third agent rewrites");
18245        assert!(msg.body.contains("`src/net.rs`"));
18246        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18247        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18248    }
18249
18250    #[tokio::test]
18251    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18252        crate::run::pin_test_home();
18253        // (agents, role): still Japanese; no TITLE; nothing resolvable.
18254        for (agents, role, saved) in [
18255            (
18256                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18257                Some("beta"),
18258                true,
18259            ),
18260            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18261            (Vec::new(), None, false),
18262        ] {
18263            let dir = tempfile::tempdir().unwrap();
18264            let mut runner = rewrite_runner(dir.path(), true, agents);
18265            runner.state.config.roles.language_judge =
18266                role.map(|r| crate::config::AgentChoice::One(r.into()));
18267            let winner = runner.state.candidates[0].clone();
18268            let first = runner
18269                .guarded_pr_message(&winner, None, true)
18270                .await
18271                .unwrap();
18272            assert!(first.is_none(), "withheld text parks the run");
18273            assert!(runner.state.parked);
18274            assert_eq!(gate_questions(&runner), 1);
18275            let g = runner.state.github_text.as_ref().unwrap();
18276            assert_eq!(g.rewrite.is_some(), saved);
18277            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18278        }
18279    }
18280
18281    #[tokio::test]
18282    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18283        crate::run::pin_test_home();
18284        let dir = tempfile::tempdir().unwrap();
18285        let mut runner = runner_at(RunStatus::Gating);
18286        runner.state =
18287            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18288        runner.state.repo = dir.path().to_owned();
18289        runner.state.candidates[0].worktree = dir.path().to_owned();
18290        let mut author = spec("alpha");
18291        author.command = vec![
18292            "sh".into(),
18293            "-c".into(),
18294            "printf '%s' \"$REWRITE_REPLY\"".into(),
18295        ];
18296        author.env.insert(
18297            "REWRITE_REPLY".into(),
18298            format!(
18299                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18300                "English sentence. ".repeat(600)
18301            ),
18302        );
18303        runner.state.config.agents = vec![author];
18304        seed_public_summary(&mut runner);
18305        let winner = runner.state.candidates[0].clone();
18306        assert!(
18307            runner
18308                .guarded_pr_message(&winner, None, true)
18309                .await
18310                .unwrap()
18311                .is_none()
18312        );
18313        let record = runner.state.github_text.clone().unwrap();
18314        let rewrite = record.rewrite.clone().unwrap();
18315        assert_eq!(rewrite.failed, ["body-language"]);
18316        assert!(
18317            rewrite
18318                .body
18319                .ends_with(&format!("magi:run/{}", runner.state.id))
18320        );
18321        let store = ask::Questions::open();
18322        let q = store.get(record.question.as_ref().unwrap()).unwrap();
18323        assert!(
18324            q.choices
18325                .contains(&crate::github_text::USE_REWRITE.to_owned())
18326        );
18327        for label in [
18328            "ORIGINAL",
18329            "FALLBACK",
18330            "REWRITE",
18331            "Rules still failed: body-language",
18332            "truncated",
18333        ] {
18334            assert!(q.detail.contains(label), "{label}");
18335        }
18336        let artifact = std::fs::read_to_string(crate::run::artifact_path(
18337            &runner.state,
18338            &crate::github_text::artifact_name(&record.fingerprint),
18339        ))
18340        .unwrap();
18341        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18342        // Restart from serialized state, then retry after a bad owner title.
18343        runner.state =
18344            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18345        store
18346            .update(&q.id, |q| {
18347                q.say("まだ日本語")?;
18348                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18349            })
18350            .unwrap();
18351        assert!(
18352            runner
18353                .guarded_pr_message(&winner, None, true)
18354                .await
18355                .unwrap()
18356                .is_none()
18357        );
18358        let retry = runner.state.github_text.as_ref().unwrap();
18359        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18360        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18361        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18362        assert!(
18363            q.posting_comparison
18364                .as_ref()
18365                .unwrap()
18366                .matches_detail(&q.detail)
18367        );
18368        store
18369            .update(&q.id, |q| {
18370                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18371            })
18372            .unwrap();
18373        let message = runner
18374            .guarded_pr_message(&winner, None, true)
18375            .await
18376            .unwrap()
18377            .unwrap();
18378        assert_eq!(message.title, rewrite.title);
18379        assert_eq!(message.body, rewrite.body);
18380        assert_eq!(
18381            runner.state.github_text.as_ref().unwrap().choice,
18382            Some(GithubTextChoice::Rewrite)
18383        );
18384        let again = runner
18385            .guarded_pr_message(&winner, None, true)
18386            .await
18387            .unwrap()
18388            .unwrap();
18389        assert_eq!(again.title, message.title);
18390        assert_eq!(again.body, message.body);
18391        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18392        assert!(
18393            !runner
18394                .state
18395                .events
18396                .iter()
18397                .any(|e| e.message.contains(&rewrite.title)
18398                    || e.message.contains("English sentence."))
18399        );
18400    }
18401
18402    #[tokio::test]
18403    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18404        crate::run::pin_test_home();
18405        let dir = tempfile::tempdir().unwrap();
18406        let mut runner = runner_at(RunStatus::Gating);
18407        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18408        runner.state.repo = dir.path().to_owned();
18409        runner.state.candidates[0].worktree = dir.path().to_owned();
18410        seed_public_summary(&mut runner);
18411        let winner = runner.state.candidates[0].clone();
18412        assert!(
18413            runner
18414                .guarded_pr_message(&winner, None, true)
18415                .await
18416                .unwrap()
18417                .is_none()
18418        );
18419        let record = runner.state.github_text.as_ref().unwrap();
18420        assert!(record.rewrite.is_none());
18421        // Records written before the additive field deserialize with no rewrite.
18422        let mut old = serde_json::to_value(record).unwrap();
18423        old.as_object_mut().unwrap().remove("rewrite");
18424        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18425        assert!(old.rewrite.is_none());
18426        let id = record.question.clone().unwrap();
18427        ask::Questions::open()
18428            .update(&id, |q| {
18429                q.choices.push(crate::github_text::USE_REWRITE.into());
18430                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18431            })
18432            .unwrap();
18433        let message = runner
18434            .guarded_pr_message(&winner, None, true)
18435            .await
18436            .unwrap()
18437            .unwrap();
18438        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18439        assert_eq!(
18440            runner.state.github_text.as_ref().unwrap().choice,
18441            Some(GithubTextChoice::Fallback)
18442        );
18443    }
18444
18445    #[tokio::test]
18446    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18447        crate::run::pin_test_home();
18448        let dir = tempfile::tempdir().unwrap();
18449        let mut runner = runner_at(RunStatus::Gating);
18450        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18451        runner.state.repo = dir.path().to_owned();
18452        runner.state.candidates[0].worktree = dir.path().to_owned();
18453        seed_public_summary(&mut runner);
18454        let winner = runner.state.candidates[0].clone();
18455        assert!(
18456            runner
18457                .guarded_pr_message(&winner, None, true)
18458                .await
18459                .unwrap()
18460                .is_none()
18461        );
18462        let store = ask::Questions::open();
18463        let find = |run: &str| {
18464            store
18465                .list()
18466                .into_iter()
18467                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18468                .unwrap()
18469        };
18470        let run = runner.state.id.clone();
18471        let q = find(&run);
18472        let fp = runner
18473            .state
18474            .github_text
18475            .as_ref()
18476            .unwrap()
18477            .fingerprint
18478            .clone();
18479        let name = crate::github_text::artifact_name(&fp);
18480        let path = crate::run::artifact_path(&runner.state, &name);
18481        assert!(q.detail.contains(&path.display().to_string()));
18482        assert!(q.detail.contains("title-language: title"));
18483        let saved = std::fs::read_to_string(&path).expect("artifact written");
18484        assert!(saved.contains("日本語のタイトル"));
18485        assert!(q.cwd.is_none());
18486        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18487        // A bad replacement is refused and asked about once more.
18488        let say = |id: &str, text: &str, choice: &str| {
18489            store
18490                .update(id, |q| {
18491                    q.thread.push(ask::Turn {
18492                        who: ask::Who::Operator,
18493                        body: text.to_owned(),
18494                        at: jiff::Timestamp::now(),
18495                        note: None,
18496                    });
18497                    q.answer(ask::Answer::Choice(choice.to_owned()))
18498                })
18499                .unwrap();
18500        };
18501        say(&q.id, "まだ日本語", "use my text");
18502        assert!(
18503            runner
18504                .guarded_pr_message(&winner, None, true)
18505                .await
18506                .unwrap()
18507                .is_none()
18508        );
18509        let second = find(&run);
18510        assert_ne!(second.id, q.id);
18511        assert!(second.detail.contains("did not pass"));
18512        // The retry still compares the original text; the rejected
18513        // replacement is named by category only.
18514        assert!(second.detail.contains("日本語のタイトル"));
18515        assert!(!second.detail.contains("まだ日本語"));
18516        assert!(second.detail.contains(&path.display().to_string()));
18517        say(&second.id, "fix: retry failed requests", "use my text");
18518        let message = runner
18519            .guarded_pr_message(&winner, None, true)
18520            .await
18521            .unwrap()
18522            .expect("a vetted replacement is posted");
18523        assert_eq!(message.title, "fix: retry failed requests");
18524        assert_eq!(
18525            runner
18526                .state
18527                .github_text
18528                .as_ref()
18529                .unwrap()
18530                .chosen_title
18531                .as_deref(),
18532            Some("fix: retry failed requests")
18533        );
18534        assert!(runner.state.github_text.as_ref().unwrap().resolved);
18535        // Resumed again: the saved decision is applied, nothing is re-asked.
18536        let again = runner
18537            .guarded_pr_message(&winner, None, true)
18538            .await
18539            .unwrap()
18540            .unwrap();
18541        assert_eq!(again.title, message.title);
18542        assert_eq!(
18543            store
18544                .list()
18545                .into_iter()
18546                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18547                .count(),
18548            2
18549        );
18550    }
18551
18552    #[test]
18553    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18554        // What magi itself writes stays English under any configured language,
18555        // so a future localisation of these headings fails here. (The agents'
18556        // own text is also checked by the posting gate.)
18557        let mut state = state_with_summary(
18558            "add retries",
18559            "TITLE: fix(web): batch reads\n- reads run.json once",
18560        );
18561        state.config.graph.language = "ja".to_owned();
18562        let m = pr_message(&state, 'A');
18563        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18564        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18565
18566        // The task is the operator's own text: it goes in untouched, and the
18567        // fallback title (no summary) may be in its language too.
18568        let task = "今回やってほしいこと: results projector を直す";
18569        let mut state = state_with_summary(task, "- no title line");
18570        state.config.graph.language = "ja".to_owned();
18571        let m = pr_message(&state, 'A');
18572        assert_eq!(
18573            m.title,
18574            format!("chore: land candidate A of run {}", state.id)
18575        );
18576        assert!(
18577            m.body.contains(&format!(
18578                "<summary>Original task</summary>\n\n{task}\n\n</details>"
18579            )),
18580            "{}",
18581            m.body
18582        );
18583    }
18584
18585    #[test]
18586    fn pr_message_scrubs_home_paths_and_addresses() {
18587        let state = state_with_summary(
18588            "fix it in /Users/someone/src/x",
18589            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18590        );
18591        let m = pr_message(&state, 'A');
18592        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18593            assert!(!m.body.contains(leak), "{}", m.body);
18594        }
18595        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18596    }
18597
18598    #[test]
18599    fn pr_message_survives_a_task_that_closes_details() {
18600        let state = state_with_summary("a </details> b", "TITLE: fix: x");
18601        let m = pr_message(&state, 'A');
18602        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18603    }
18604
18605    #[test]
18606    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18607        let cmd = manual_merge_command(
18608            MergeStyle::Squash,
18609            Path::new("/repo"),
18610            "b",
18611            "fix: \"quoted\" $(x) `y`\n\nbody",
18612        );
18613        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18614    }
18615
18616    #[test]
18617    fn manual_merge_command_matches_the_configured_style() {
18618        let repo = Path::new("/repo");
18619        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18620
18621        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18622        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18623
18624        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18625        assert_eq!(
18626            squash,
18627            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18628             \"Merge magi run 0832 (candidate A)\""
18629        );
18630
18631        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18632        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18633    }
18634
18635    #[test]
18636    fn a_nudge_gets_a_quarter_of_the_budget() {
18637        // The judge and implement budgets magi ships with.
18638        assert_eq!(retry_budget(secs(1200), true), secs(300));
18639        assert_eq!(retry_budget(secs(3600), true), secs(900));
18640    }
18641
18642    #[test]
18643    fn a_resent_prompt_keeps_the_whole_budget() {
18644        // The seat kept no context, so the retry is the original job again and
18645        // shortening it would only guarantee a second failure.
18646        assert_eq!(retry_budget(secs(1200), false), secs(1200));
18647        assert_eq!(retry_budget(secs(60), false), secs(60));
18648    }
18649
18650    #[test]
18651    fn the_floor_never_exceeds_the_original_budget() {
18652        // A short configured timeout must not be *raised* by the floor: the
18653        // operator asked for a bound, and a retry may not outlast the attempt
18654        // it is retrying.
18655        assert_eq!(retry_budget(secs(60), true), secs(60));
18656        assert_eq!(retry_budget(secs(480), true), secs(120));
18657        assert_eq!(retry_budget(secs(0), true), secs(0));
18658    }
18659
18660    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18661        agent::CommandEvidence {
18662            id: "item1".to_owned(),
18663            description: "cargo test".to_owned(),
18664            exit_code,
18665            result_summary: String::new(),
18666            source: "codex".to_owned(),
18667        }
18668    }
18669
18670    #[test]
18671    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18672        // No evidence is not the same fact as unconfirmed evidence: a
18673        // backend with no adapter, or a reply that ran no commands at all,
18674        // must not be misread as carrying a dangling job.
18675        assert!(!has_unconfirmed_command(&[]));
18676    }
18677
18678    #[test]
18679    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18680        // Deliberately not a check on the exit code's *value*: a fixer
18681        // legitimately runs something that fails mid-iteration before it
18682        // succeeds, and that must never by itself reopen a valid report.
18683        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18684        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18685        assert!(!has_unconfirmed_command(&[
18686            evidence(Some(0)),
18687            evidence(Some(101))
18688        ]));
18689    }
18690
18691    #[test]
18692    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18693        assert!(has_unconfirmed_command(&[
18694            evidence(Some(0)),
18695            evidence(None)
18696        ]));
18697    }
18698
18699    #[test]
18700    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18701        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18702        assert_eq!(
18703            verified_noop_claim(true, &[], text).as_deref(),
18704            Some("already fixed by b32cfc4, on main.")
18705        );
18706    }
18707
18708    #[test]
18709    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18710        // A timeout or a bad exit code reads as the ordinary loss it is,
18711        // whatever the reply's own prose claims.
18712        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18713        assert!(verified_noop_claim(false, &[], text).is_none());
18714    }
18715
18716    #[test]
18717    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18718        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18719        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18720        // A confirmed command alongside the marker is fine.
18721        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18722    }
18723
18724    #[test]
18725    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18726        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18727    }
18728
18729    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
18730    /// pair, in order, labelled A, B, C, ...
18731    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18732        runner.state.candidates = shape
18733            .iter()
18734            .enumerate()
18735            .map(|(i, &(empty, verified))| Candidate {
18736                index: i,
18737                label: (b'A' + i as u8) as char,
18738                agent: "sonnet".to_owned(),
18739                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18740                worktree: PathBuf::from(format!("/wt/{i}")),
18741                summary: String::new(),
18742                stat: String::new(),
18743                files: 0,
18744                commits: 0,
18745                empty,
18746                failed: None,
18747                verified_noop: verified.map(str::to_owned),
18748                duration_ms: 0,
18749                folded: false,
18750            })
18751            .collect();
18752    }
18753
18754    #[test]
18755    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18756        ask_test_home();
18757        let mut runner = runner_at(RunStatus::Implementing);
18758        set_candidates(
18759            &mut runner,
18760            &[
18761                (true, Some("already on main at b32cfc4")),
18762                (true, Some("same fix, see the existing test")),
18763            ],
18764        );
18765
18766        runner
18767            .after_implement()
18768            .expect("a verified no-op is not an error");
18769
18770        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18771    }
18772
18773    #[test]
18774    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18775        ask_test_home();
18776        let mut runner = runner_at(RunStatus::Implementing);
18777        // Candidate A declares a verified no-op; candidate B simply wrote
18778        // nothing and said nothing about why. One candidate's claim is not
18779        // the whole run's agreement.
18780        set_candidates(
18781            &mut runner,
18782            &[(true, Some("already on main at b32cfc4")), (true, None)],
18783        );
18784
18785        let err = runner
18786            .after_implement()
18787            .expect_err("an unverified empty candidate must still fail the run");
18788
18789        assert!(
18790            err.to_string().contains("no candidate produced a change"),
18791            "{err}"
18792        );
18793        assert_eq!(runner.state.status, RunStatus::Failed);
18794    }
18795
18796    #[test]
18797    fn after_implement_still_fails_an_ordinary_all_empty_run() {
18798        ask_test_home();
18799        let mut runner = runner_at(RunStatus::Implementing);
18800        set_candidates(&mut runner, &[(true, None), (true, None)]);
18801
18802        let err = runner
18803            .after_implement()
18804            .expect_err("no candidate declared anything; this is an ordinary failure");
18805
18806        assert!(
18807            err.to_string().contains("no candidate produced a change"),
18808            "{err}"
18809        );
18810        assert_eq!(runner.state.status, RunStatus::Failed);
18811    }
18812}
18813
18814/// A graft rollback whose restore could not be confirmed (see
18815/// [`Runner::graft`]).
18816#[derive(Debug)]
18817struct GraftRestoreFailed(String);
18818
18819impl std::fmt::Display for GraftRestoreFailed {
18820    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18821        write!(
18822            f,
18823            "could not restore the winner to its pre-graft commit: {}",
18824            self.0
18825        )
18826    }
18827}
18828
18829impl std::error::Error for GraftRestoreFailed {}
18830
18831/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
18832/// not the agent: they never draw on an agent's own timeout, and running out of
18833/// them never reads as an agent failure.
18834#[cfg(test)]
18835mod resource_wait_tests {
18836    use super::*;
18837    use std::collections::BTreeMap;
18838
18839    fn sh(id: &str, script: &str) -> AgentSpec {
18840        AgentSpec {
18841            id: id.to_owned(),
18842            kind: crate::config::AgentKind::Command,
18843            model: None,
18844            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18845            extra_args: Vec::new(),
18846            env: BTreeMap::new(),
18847            prompt_delivery: None,
18848            max_turn_secs: None,
18849        }
18850    }
18851
18852    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18853        SeatJob {
18854            seat: SeatState::new("impl-A", &spec.id, 1),
18855            spec,
18856            cwd: dir.to_path_buf(),
18857            prompt: "do it".to_owned(),
18858            timeout,
18859            allow_write: true,
18860            sessions: false,
18861            artifacts: dir.join("art"),
18862            stem: "impl-A".to_owned(),
18863            handover: None,
18864        }
18865    }
18866
18867    fn state(resource_wait_secs: u64) -> RunState {
18868        let mut config = Config::default();
18869        config.graph.timeout_resource_wait = resource_wait_secs;
18870        RunState::new(
18871            PathBuf::from("/repo"),
18872            "main".to_owned(),
18873            "abc1234".to_owned(),
18874            "task".to_owned(),
18875            config,
18876        )
18877    }
18878
18879    fn hold(cache: &Path) -> crate::cache::Guard {
18880        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18881        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18882            crate::cache::AcquireOutcome::Acquired(g) => g,
18883            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18884        }
18885    }
18886
18887    async fn run(
18888        jobs: Vec<SeatJob>,
18889        sem: Arc<Semaphore>,
18890        cache: Option<&Path>,
18891        st: &mut RunState,
18892    ) -> Vec<(usize, SeatState, AgentOutcome)> {
18893        let prompts = Prompts::default();
18894        let ctx = WaveCtx {
18895            run: "r",
18896            node: "implement",
18897            prompts: &prompts,
18898            cache,
18899            round: None,
18900            carry_seats: false,
18901            pauses: None,
18902        };
18903        wave(jobs, sem, &ctx, st, 0).await
18904    }
18905
18906    #[tokio::test]
18907    async fn verification_cache_wait_preserves_the_command_budget() {
18908        crate::run::pin_test_home();
18909        let tmp = tempfile::tempdir().unwrap();
18910        let cache = tmp.path().join("target");
18911        let guard = hold(&cache);
18912        let release = tokio::spawn(async move {
18913            tokio::time::sleep(Duration::from_millis(100)).await;
18914            guard.release();
18915        });
18916        let mut st = state(10);
18917        let budget = Duration::from_millis(50);
18918        let wait = repository_wait(&st, None);
18919        let (outcomes, _) = with_cache_lease(
18920            &mut st,
18921            Some(&cache),
18922            "gate",
18923            "gate",
18924            tmp.path(),
18925            "abc1234",
18926            budget,
18927            "test gate",
18928            &wait,
18929            |_, available| async move {
18930                assert_eq!(
18931                    available, budget,
18932                    "contention must not shorten verification"
18933                );
18934                (Vec::new(), false, Vec::new())
18935            },
18936        )
18937        .await
18938        .unwrap();
18939        release.await.unwrap();
18940        assert!(outcomes.is_empty(), "{outcomes:?}");
18941        assert!(
18942            st.events
18943                .iter()
18944                .any(|event| event.message.contains("waiting for"))
18945        );
18946        assert!(st.resource_wait.is_none());
18947        // The verification body no longer owns the lease after returning.
18948        hold(&cache).release();
18949    }
18950
18951    #[tokio::test]
18952    async fn verification_cache_wait_expires_without_starting_commands() {
18953        crate::run::pin_test_home();
18954        let tmp = tempfile::tempdir().unwrap();
18955        let cache = tmp.path().join("target");
18956        let _held = hold(&cache);
18957        let mut st = state(0);
18958        let wait = repository_wait(&st, None);
18959        let (outcomes, _) = with_cache_lease(
18960            &mut st,
18961            Some(&cache),
18962            "gate",
18963            "gate",
18964            tmp.path(),
18965            "abc1234",
18966            Duration::from_secs(60),
18967            "test gate",
18968            &wait,
18969            |_, _| async { panic!("commands must not start without the cache lease") },
18970        )
18971        .await
18972        .unwrap();
18973        assert_eq!(outcomes.len(), 1);
18974        assert!(outcomes[0].resource_blocked);
18975        assert!(st.resource_wait.is_none());
18976    }
18977
18978    #[tokio::test]
18979    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18980        crate::run::pin_test_home();
18981        let tmp = tempfile::tempdir().unwrap();
18982        let cache = tmp.path().join("target-a");
18983        let guard = hold(&cache);
18984        // Held for longer than the agent's whole 1s budget.
18985        tokio::spawn(async move {
18986            tokio::time::sleep(Duration::from_millis(1500)).await;
18987            guard.release();
18988        });
18989        let mut st = state(60);
18990        let results = run(
18991            vec![job(
18992                sh("alpha", "echo ok"),
18993                tmp.path(),
18994                Duration::from_secs(1),
18995            )],
18996            Arc::new(Semaphore::new(1)),
18997            Some(&cache),
18998            &mut st,
18999        )
19000        .await;
19001        assert!(
19002            matches!(results[0].2, AgentOutcome::Ok(_)),
19003            "the wait must not be charged to the agent"
19004        );
19005        assert!(st.resource_block.is_none());
19006        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19007        assert!(st.resource_wait.is_none() && st.active.is_empty());
19008    }
19009
19010    #[tokio::test]
19011    #[cfg(unix)]
19012    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19013        use std::os::fd::AsRawFd as _;
19014        crate::run::pin_test_home();
19015        let tmp = tempfile::tempdir().unwrap();
19016        crate::git::git(tmp.path(), &["init"]).await.unwrap();
19017        let common = tmp.path().join(".git");
19018        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19019        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19020        assert_eq!(
19021            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19022            0
19023        );
19024        let mut st = state(1);
19025        st.config.coordination = crate::config::test_coordination();
19026        let mut reader = job(
19027            sh("reader", "echo must-not-run"),
19028            tmp.path(),
19029            Duration::from_secs(30),
19030        );
19031        reader.allow_write = false;
19032        let results = run(
19033            vec![reader.clone()],
19034            Arc::new(Semaphore::new(1)),
19035            None,
19036            &mut st,
19037        )
19038        .await;
19039        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19040        assert_eq!(results[0].1.turns, 0);
19041        assert!(FailClass::of(&results[0].2).is_none());
19042        assert_eq!(
19043            st.resource_block.as_ref().unwrap().kind,
19044            "repository coordination"
19045        );
19046        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19047        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19048        drop(external);
19049        let editor = crate::repo_lock::SeatLease::for_checkout(
19050            tmp.path(),
19051            true,
19052            crate::config::test_coordination().as_ref(),
19053        )
19054        .await
19055        .unwrap();
19056        let pause = Pause::new();
19057        let interrupt = Pause::new();
19058        let prompts = Prompts::default();
19059        let ctx = WaveCtx {
19060            run: "r",
19061            node: "review",
19062            prompts: &prompts,
19063            cache: None,
19064            round: None,
19065            carry_seats: false,
19066            pauses: Some((&pause, &interrupt)),
19067        };
19068        let (results, ()) = tokio::join!(
19069            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19070            async {
19071                tokio::time::sleep(Duration::from_millis(50)).await;
19072                pause.park();
19073            }
19074        );
19075        assert!(
19076            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19077        );
19078        assert_eq!(results[0].1.turns, 0);
19079        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19080        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19081        drop(editor);
19082    }
19083
19084    #[tokio::test]
19085    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19086        crate::run::pin_test_home();
19087        let tmp = tempfile::tempdir().unwrap();
19088        let cache = tmp.path().join("target-b");
19089        let _held = hold(&cache);
19090        let mut st = state(1);
19091        let results = run(
19092            vec![job(
19093                sh("alpha", "echo ok"),
19094                tmp.path(),
19095                Duration::from_secs(1),
19096            )],
19097            Arc::new(Semaphore::new(1)),
19098            Some(&cache),
19099            &mut st,
19100        )
19101        .await;
19102        let (_, seat, out) = &results[0];
19103        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19104        assert!(wave_blocked(&results));
19105        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19106        assert_eq!(seat.turns, 0, "nothing was asked");
19107        assert!(
19108            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19109            "no agent was spawned"
19110        );
19111        assert!(FailClass::of(out).is_none(), "not a failure class");
19112        let block = st.resource_block.as_ref().expect("recorded for the resume");
19113        assert_eq!(block.kind, "shared build cache");
19114        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19115        assert!(st.active.is_empty());
19116    }
19117
19118    #[tokio::test]
19119    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19120        crate::run::pin_test_home();
19121        let tmp = tempfile::tempdir().unwrap();
19122        let mut st = state(1);
19123        let results = run(
19124            vec![job(
19125                sh("alpha", "echo ok"),
19126                tmp.path(),
19127                Duration::from_secs(1),
19128            )],
19129            Arc::new(Semaphore::new(0)),
19130            None,
19131            &mut st,
19132        )
19133        .await;
19134        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19135        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19136        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19137    }
19138
19139    #[tokio::test]
19140    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19141        crate::run::pin_test_home();
19142        let tmp = tempfile::tempdir().unwrap();
19143        let mut st = state(60);
19144        let results = run(
19145            vec![job(
19146                sh("alpha", "sleep 30"),
19147                tmp.path(),
19148                Duration::from_secs(1),
19149            )],
19150            Arc::new(Semaphore::new(1)),
19151            None,
19152            &mut st,
19153        )
19154        .await;
19155        let out = &results[0].2;
19156        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19157        let class = FailClass::of(out).expect("a failure");
19158        assert_eq!(class, FailClass::Timeout);
19159        assert!(should_hand_over(None, &class), "existing handover policy");
19160        assert!(st.resource_block.is_none());
19161    }
19162}