Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's network, Git access and repository coordination before \
526         retrying"
527    )
528}
529
530/// Outcome of the shared fixer chain, before caller-owned stop policy.
531enum FixStep {
532    Resource,
533    Recorded { report_lost: bool },
534}
535
536/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
537/// on an early return or a panic.
538///
539/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
540/// manual `magi fix` invocations against the same run are otherwise
541/// invisible to each other and would race to remove and recreate the same
542/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
543/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
544/// which is only ever reclaimed later, out of band, by
545/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
546/// `magi fix` invocation is not necessarily running under either of those, so
547/// nothing would ever sweep a lock a killed or crashed process left behind.
548/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
549/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
550/// lease use: an unreadable or unparsable pid, or a liveness query the
551/// platform cannot answer, reads as alive and the lock is left in place.
552struct FixClaim {
553    path: PathBuf,
554}
555
556impl FixClaim {
557    fn acquire(dir: &Path) -> Result<Self> {
558        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559        let path = dir.join("fix.lock");
560        let retry = crate::lock_file::Retry::new(None);
561        match Self::create(&path, retry) {
562            Ok(claim) => Ok(claim),
563            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564                if Self::reclaim_if_dead(&path) {
565                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566                } else {
567                    bail!(
568                        "another repair or `magi fix` is already running for this run ({} exists)",
569                        path.display()
570                    )
571                }
572            }
573            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574        }
575    }
576
577    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578        let mut f = retry.create(path)?;
579        use std::io::Write as _;
580        // Read back by `reclaim_if_dead` on a later, stuck invocation.
581        writeln!(f, "{}", std::process::id())?;
582        Ok(Self {
583            path: path.to_owned(),
584        })
585    }
586
587    /// True if the lock named a process confirmed dead, in which case it was
588    /// also removed. Never true on an unreadable file, an unparsable pid, or
589    /// a liveness query the platform cannot answer — see this type's own doc.
590    fn reclaim_if_dead(path: &Path) -> bool {
591        let dead = std::fs::read_to_string(path)
592            .ok()
593            .and_then(|body| body.trim().parse::<u32>().ok())
594            .is_some_and(|pid| !crate::proc::pid_alive(pid));
595        dead && std::fs::remove_file(path).is_ok()
596    }
597}
598
599impl Drop for FixClaim {
600    fn drop(&mut self) {
601        let _ = std::fs::remove_file(&self.path);
602    }
603}
604
605/// Holds the mutable runner as well as its durable lease: cancellation must
606/// clear both copies of ownership, including state a caller may save later.
607struct DriverExecution<'a> {
608    runner: &'a mut Runner,
609    driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613    fn drop(&mut self) {
614        self.driver.release_local(&mut self.runner.state);
615    }
616}
617
618impl Runner {
619    /// Start a fresh run against `repo`.
620    pub async fn start(
621        repo: &Path,
622        instruction: String,
623        config: Config,
624        origin: Origin,
625    ) -> Result<Self> {
626        Self::start_naming(repo, instruction, "", config, origin).await
627    }
628
629    /// [`Runner::start`] for a queued task. Titles and instructions never
630    /// select implementation sources. The title argument is retained for callers.
631    pub async fn start_naming(
632        repo: &Path,
633        instruction: String,
634        _also_scan: &str,
635        config: Config,
636        origin: Origin,
637    ) -> Result<Self> {
638        let repo = git::toplevel(repo).await?;
639        let missing = agent::missing_programs(&config.agents);
640        if !missing.is_empty() {
641            bail!(
642                "these agent programs are not on PATH: {}. Fix the roster in \
643                 magi.toml or install them.",
644                missing.join(", ")
645            );
646        }
647        let base_branch =
648            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649                .await?;
650        let fetched_base = resolve_base(
651            &repo,
652            &base_branch,
653            &config.merge.remote,
654            config.coordination.as_ref(),
655        )
656        .await?;
657        let base_commit = fetched_base.commit.clone();
658        let roles = config.resolve_roles()?;
659        let max_parallel = config.graph.max_parallel.max(1);
660        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661        // Recorded before the first save, so a crash right after minting
662        // cannot leave a run with no origin. Legibility only: nothing reads it
663        // to decide anything.
664        state.fetched_base = Some(fetched_base);
665        state.origin = Some(origin);
666        state.reserve()?;
667        state.event("start", format!("run {} created", state.id));
668        state.save()?;
669        Ok(Self {
670            state,
671            roles,
672            sem: Arc::new(Semaphore::new(max_parallel)),
673            pause: Pause::new(),
674            interrupt: Pause::new(),
675        })
676    }
677
678    /// Open a review-only run against work that already exists on `branch`.
679    ///
680    /// The expensive half of the graph is the implement wave — measured at
681    /// 111 and 134 internal tool-loop turns on this repository, against a
682    /// handful for a judge or a reviewer. The cheap half is worth running on
683    /// hand-written work too, and there was no way to reach it.
684    ///
685    /// No new state and no schema change are needed: a run with **one** viable
686    /// candidate and a tally already decided degrades `execute` to exactly
687    /// review → gate → merge, because `judge` skips a single-candidate field,
688    /// `deliberate` has fewer than two first choices to reconcile, `vote`
689    /// returns early, `tally` is already present and `fold_losers` has no
690    /// losers. Resuming such a run therefore does the right thing as well.
691    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692        Self::review_taking_over(repo, branch, config, None, origin).await
693    }
694
695    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
696    /// at the same task still has `branch` checked out, its worktree is
697    /// released first if that is safe (see [`crate::handover`]), and the
698    /// review refuses with the reason if it is not. `None` is a hand-run
699    /// review: it has no earlier attempts, so only a worktree of a dead run
700    /// magi recorded itself can be released.
701    pub async fn review_taking_over(
702        repo: &Path,
703        branch: &str,
704        config: Config,
705        takeover: Option<crate::handover::Takeover>,
706        origin: Origin,
707    ) -> Result<Self> {
708        let repo = git::toplevel(repo).await?;
709        let missing = agent::missing_programs(&config.agents);
710        if !missing.is_empty() {
711            bail!(
712                "these agent programs are not on PATH: {}. Fix the roster in \
713                 magi.toml or install them.",
714                missing.join(", ")
715            );
716        }
717        let base_branch =
718            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719                .await?;
720        if base_branch == branch {
721            bail!("`{branch}` is the base branch; there is nothing to review against");
722        }
723        let fetched_base = resolve_base(
724            &repo,
725            &base_branch,
726            &config.merge.remote,
727            config.coordination.as_ref(),
728        )
729        .await?;
730        let base_commit = fetched_base.commit.clone();
731
732        let roles = config.resolve_roles()?;
733        let max_parallel = config.graph.max_parallel.max(1);
734        let mut state = RunState::new(
735            repo.clone(),
736            base_branch,
737            base_commit.clone(),
738            String::new(),
739            config,
740        );
741        state.fetched_base = Some(fetched_base);
742        state.origin = Some(origin);
743        state.reserve()?;
744
745        // Released before anything else touches the branch: a stale local
746        // branch is moved with `git branch -f`, which git refuses while an
747        // earlier attempt's worktree still has it checked out. Everything
748        // after this point that can fail puts the old run back.
749        // A hand-run review has no task, hence no earlier attempts, but a
750        // worktree of a dead run magi made may still be released.
751        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752            earlier: Vec::new(),
753            home: crate::run::home(),
754            choice: None,
755            task_instruction: None,
756        });
757        let wait = repository_wait(&state, None);
758        let coordination_config = state.config.coordination.clone();
759        let coordination =
760            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761                .await?;
762        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763        if let Some(released) = &released {
764            state.event(
765                "release",
766                format!(
767                    "took `{branch}` over from run {}: its worktree was released: {}",
768                    crate::run::short_of(&released.old_id),
769                    released.audit
770                ),
771            );
772        }
773        // The owner's answer to an earlier divergence question is applied
774        // here: after the release (git will not move a checked-out branch)
775        // and before the sync that would otherwise ask again.
776        if let Some(choice) = takeover.choice.as_ref()
777            && let Err(e) =
778                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779                    .await
780        {
781            if let Some(released) = &released {
782                released.restore(&repo, branch).await;
783            }
784            return Err(e.context("applying the owner's answer about the diverged branch"));
785        }
786        // An earlier attempt of this task that published the task head, and
787        // whose work is the branch under review, hands that head over: the
788        // review run updates the same pull request. Without one (a legacy
789        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
790        // own head and its own open pull request is adopted as before.
791        for id in takeover.earlier.iter().rev() {
792            if let Ok(old) = RunState::load_under(id, &takeover.home)
793                && let Some(head) = old.pr_head.clone()
794                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795            {
796                state.pr_head = Some(head);
797                break;
798            }
799        }
800        drop(coordination);
801        let opened = Self::open_review(
802            &repo,
803            branch,
804            state,
805            roles,
806            max_parallel,
807            base_commit,
808            &takeover,
809        )
810        .await;
811        if opened.is_err()
812            && let Some(released) = &released
813        {
814            let _coordination = crate::repo_lock::SeatLease::operation_with(
815                &repo,
816                coordination_config.as_ref(),
817                &wait,
818            )
819            .await?;
820            released.restore(&repo, branch).await;
821        }
822        opened
823    }
824
825    /// Recover exactly the operator-selected public implementation, without touching old sources.
826    pub async fn recover_pr(
827        repo: &Path,
828        task: &str,
829        selection: crate::pr_recovery::Selection,
830        mut config: Config,
831    ) -> Result<Self> {
832        selection.configure(&mut config)?;
833        selection.check_task(task, &selection.instruction)?;
834        let base_branch =
835            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836                .await?;
837        let fetched_base = resolve_base(
838            repo,
839            &base_branch,
840            &config.merge.remote,
841            config.coordination.as_ref(),
842        )
843        .await?;
844        let base_commit = fetched_base.commit.clone();
845        selection
846            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847            .await?;
848        Box::pin(selection.verify_source(repo)).await?;
849        if let Some(proof) = &selection.source_proof {
850            anyhow::ensure!(
851                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852                "source rebase base is outside the fetched landing base"
853            );
854        }
855        let roles = config.resolve_roles()?;
856        let max_parallel = config.graph.max_parallel.max(1);
857        let mut state = RunState::new(
858            repo.to_path_buf(),
859            base_branch,
860            base_commit.clone(),
861            selection.instruction.clone(),
862            config,
863        );
864        state.fetched_base = Some(fetched_base);
865        state.origin = Some(crate::run::Origin::queue(task));
866        state.pr_head = Some(selection.branch.clone());
867        state.pr_recovery = Some(selection.clone());
868        state.reserve()?;
869        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870            // Never leave a reserved record behind as a phantom owner of the head.
871            state.discard_reserved();
872            return Err(e);
873        }
874        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875    }
876
877    async fn open_selected(
878        repo: &Path,
879        state: RunState,
880        roles: ResolvedRoles,
881        max_parallel: usize,
882        base_commit: String,
883        selection: crate::pr_recovery::Selection,
884    ) -> Result<Self> {
885        let mut state = state;
886        let branch = state.branch_for('A');
887        let wait = repository_wait(&state, None);
888        let prepared = async {
889            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890            git::git(repo, &["branch", &branch, selection.source()]).await?;
891            drop(coordination);
892            anyhow::Ok(())
893        }
894        .await;
895        if let Err(e) = prepared {
896            // Nothing was exposed yet: do not leave a reserved record as a
897            // phantom owner of the public head.
898            state.discard_reserved();
899            return Err(e);
900        }
901        state.event(
902            "recovery",
903            format!(
904                "selected implementation {} for {} (public lease {})",
905                selection.source(),
906                selection.url,
907                selection.head
908            ),
909        );
910        let takeover = crate::handover::Takeover {
911            earlier: Vec::new(),
912            home: crate::run::home(),
913            choice: None,
914            task_instruction: Some(selection.instruction.clone()),
915        };
916        if let Err(e) = state.save() {
917            state.discard_reserved();
918            return Err(e);
919        }
920        let id = state.id.clone();
921        let opened = Self::open_review(
922            repo,
923            &branch,
924            state,
925            roles,
926            max_parallel,
927            base_commit,
928            &takeover,
929        )
930        .await;
931        match opened {
932            Ok(runner) => Ok(runner),
933            Err(e) => {
934                if let Ok(mut partial) = RunState::load(&id) {
935                    partial.status = RunStatus::Blocked;
936                    partial.driver_exited = true;
937                    partial.event(
938                        "recovery",
939                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
940                    );
941                    partial.save()?;
942                }
943                Err(crate::pr_recovery::StartupFailed {
944                    run: id,
945                    cause: format!("{e:#}"),
946                }
947                .into())
948            }
949        }
950    }
951
952    /// What the newest earlier attempt that reviewed `branch` leaves for this
953    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
954    /// no earlier attempts. Reads the earlier record only: ownership of its
955    /// worktree and branch is [`crate::handover`]'s business.
956    async fn inherit(
957        repo: &Path,
958        branch: &str,
959        base_commit: &str,
960        takeover: &crate::handover::Takeover,
961    ) -> Option<crate::recovery::RecoveryRecord> {
962        use crate::recovery::{Relation, assess};
963        let newest = takeover.earlier.last()?;
964        let mut chosen = None;
965        for id in takeover.earlier.iter().rev() {
966            if let Ok(old) = RunState::load_under(id, &takeover.home)
967                && !old.reviews.is_empty()
968                && (old.winner().is_some_and(|w| w.branch == branch)
969                    || old.pr_head.as_deref() == Some(branch))
970            {
971                chosen = Some(old);
972                break;
973            }
974        }
975        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976            .await
977            .unwrap_or_default();
978        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979            Some(last) if last.head == tip => Relation::Same,
980            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981            _ => Relation::Diverged,
982        };
983        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984        Some(assess(from, chosen.as_ref(), base_commit, relation))
985    }
986
987    /// The half of [`Runner::review_taking_over`] that can fail after an
988    /// earlier attempt's worktree was released.
989    async fn open_review(
990        repo: &Path,
991        branch: &str,
992        mut state: RunState,
993        roles: ResolvedRoles,
994        max_parallel: usize,
995        base_commit: String,
996        takeover: &crate::handover::Takeover,
997    ) -> Result<Self> {
998        let wait = repository_wait(&state, None);
999        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000        if state.pr_recovery.is_none() {
1001            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002        }
1003        // The commit subjects are the closest thing to a task statement that
1004        // existing work carries, and the reviewers are told as much.
1005        let start = review_base(
1006            repo,
1007            &state.config.merge.remote,
1008            &state.base_branch,
1009            &base_commit,
1010            branch,
1011        )
1012        .await;
1013        let log = git::log_oneline(repo, &start, branch)
1014            .await
1015            .unwrap_or_default();
1016        let log_text = if log.trim().is_empty() {
1017            "(no commit messages)"
1018        } else {
1019            log.trim()
1020        };
1021        let mut instruction = format!(
1022            "Review the work already on branch `{branch}`. There is no task \
1023             statement: what the change claims to do is whatever its commits \
1024             say.\n\n{log_text}"
1025        );
1026        // A retry of a task whose earlier attempt left reviews behind carries
1027        // that attempt's evidence, and the task's own statement with it: the
1028        // completion criteria are the task's, not whatever the commits claim.
1029        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030            if let Some(task) = takeover
1031                .task_instruction
1032                .as_deref()
1033                .filter(|t| !t.trim().is_empty())
1034                && rec.reviewed_head.is_some()
1035            {
1036                instruction = format!(
1037                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038                    task.trim()
1039                );
1040                rec.criteria_inherited = true;
1041            }
1042            state.event("recovery", rec.summary());
1043            state.recovery = Some(rec);
1044        }
1045        if let Some(selected) = &state.pr_recovery {
1046            instruction = selected.instruction.clone();
1047        }
1048        state.task_instruction = takeover.task_instruction.clone();
1049        state.instruction = instruction;
1050        state.reviewed_commits = Some(
1051            git::subjects(repo, &start, branch)
1052                .await
1053                .unwrap_or_default(),
1054        );
1055
1056        // A takeover must not forget a hold that a previous review observed,
1057        // even if repairs have since removed it from the branch.
1058        for id in &takeover.earlier {
1059            if let Ok(old) = RunState::load_under(id, &takeover.home)
1060                && old.repo == state.repo
1061                && (old.pr_head.as_deref() == Some(branch)
1062                    || old.winner().is_some_and(|w| w.branch == branch))
1063            {
1064                for hold in old.branch_owner_holds {
1065                    if !state.branch_owner_holds.contains(&hold) {
1066                        state.event(
1067                            "approval-policy",
1068                            format!(
1069                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070                                hold.control.key(),
1071                                hold.file,
1072                                hold.head
1073                            ),
1074                        );
1075                        state.branch_owner_holds.push(hold);
1076                    }
1077                }
1078            }
1079        }
1080        let reviewed_head = git::rev_parse(repo, branch).await?;
1081        state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083        // An attached worktree, so the fixer's commits land on the branch under
1084        // review rather than on a detached head nobody will look at again.
1085        let worktree = state.worktree_root().join("under-review");
1086        if let Some(parent) = worktree.parent() {
1087            tokio::fs::create_dir_all(parent).await.ok();
1088        }
1089        let path = worktree.to_string_lossy().to_string();
1090        git::git(repo, &["worktree", "add", &path, branch])
1091            .await
1092            .with_context(|| {
1093                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094            })?;
1095
1096        let setup = match ensure_setup_config(&mut state, repo).await {
1097            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098            Err(e) => Err(e),
1099        };
1100        if let Err(e) = setup {
1101            git::worktree_remove_clean(repo, &worktree).await.ok();
1102            return Err(e);
1103        }
1104        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105            .await
1106            .unwrap_or(0);
1107        if commits == 0 {
1108            git::worktree_remove_clean(repo, &worktree).await.ok();
1109            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110        }
1111        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112            .await
1113            .map(|f| f.len())
1114            .unwrap_or(0);
1115        if files == 0
1116            && let (Ok(head_tree), Ok(base_tree)) = (
1117                git::tree_of(&worktree, "HEAD").await,
1118                git::tree_of(&worktree, &base_commit).await,
1119            )
1120            && head_tree == base_tree
1121        {
1122            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123            git::worktree_remove_clean(repo, &worktree).await.ok();
1124            bail!(
1125                "`{branch}` at {} has a tree identical to base {}; this usually means \
1126                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127                 against `{}/{branch}`) rather than an empty change",
1128                short(&head),
1129                short(&base_commit),
1130                state.config.merge.remote
1131            );
1132        }
1133        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134            .await
1135            .unwrap_or_default();
1136
1137        state.candidates.push(Candidate {
1138            index: 0,
1139            label: 'A',
1140            // Not an agent id on purpose: nothing in the roster wrote this, and
1141            // the stats tables must not credit anyone with a win for it.
1142            agent: EXISTING_BRANCH.to_owned(),
1143            branch: branch.to_owned(),
1144            worktree,
1145            summary: String::new(),
1146            stat,
1147            files,
1148            commits,
1149            empty: false,
1150            failed: None,
1151            verified_noop: None,
1152            duration_ms: 0,
1153            folded: false,
1154        });
1155        state.tally = Some(Tally {
1156            first_choice: BTreeMap::from([('A', 0)]),
1157            borda: BTreeMap::new(),
1158            winner: 'A',
1159            rankings: 0,
1160            unanimous_initial: false,
1161            deliberated: false,
1162            changed_votes: 0,
1163            unanimous_final: false,
1164            tie_break: None,
1165            // No panel sat, so no quorum applies. Zero judges is the correct
1166            // number for work that never competed, and must not be reported as
1167            // a collapsed panel.
1168            judges: 0,
1169            present: 0,
1170            quorum: 0,
1171            met_quorum: true,
1172            uncontested: Some("review-only run: nothing competed".to_owned()),
1173        });
1174        state.status = RunStatus::Reviewing;
1175        state.prep_complete = Some(true);
1176        state.event(
1177            "start",
1178            format!(
1179                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180                state.id
1181            ),
1182        );
1183        state.save()?;
1184        Ok(Self {
1185            state,
1186            roles,
1187            sem: Arc::new(Semaphore::new(max_parallel)),
1188            pause: Pause::new(),
1189            interrupt: Pause::new(),
1190        })
1191    }
1192
1193    /// Reopen an existing run.
1194    pub fn resume(id: &str) -> Result<Self> {
1195        let state = RunState::load(id)?;
1196        crate::pr_head::ensure_owner(&state)?;
1197        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198            bail!(
1199                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200            );
1201        }
1202        if let Some(to) = &state.released_to {
1203            bail!(
1204                "run {} cannot be resumed: its worktree was released to run {}",
1205                state.short(),
1206                crate::run::short_of(to)
1207            );
1208        }
1209        let roles = state.config.resolve_roles()?;
1210        if let Some(id) = &state.config.graph.selected_implementer
1211            && state.candidates.iter().any(|c| c.agent != *id)
1212        {
1213            bail!(
1214                "replacement implementer pin does not match the saved candidates; task must stay held"
1215            );
1216        }
1217        let max_parallel = state.config.graph.max_parallel.max(1);
1218        Ok(Self {
1219            state,
1220            roles,
1221            sem: Arc::new(Semaphore::new(max_parallel)),
1222            pause: Pause::new(),
1223            interrupt: Pause::new(),
1224        })
1225    }
1226
1227    /// Walk the graph to a terminal state, skipping nodes already recorded.
1228    ///
1229    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1230    /// the phone - ends here, so this is the one place a run that ended
1231    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1232    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1233    pub async fn execute(&mut self) -> Result<()> {
1234        if let Some(selection) = &self.state.pr_recovery
1235            && (selection.source_proof.is_some() || selection.source_run.is_some())
1236        {
1237            Box::pin(selection.verify_source(&self.state.repo)).await?;
1238        }
1239        // Queue claims cover daemon work; this also serializes manual drivers
1240        // consuming a repair answer, including the new review and decision.
1241        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242            || self.state.land_approval.is_some()
1243            || self.state.merge_decision.is_some()
1244            || !self.state.merge_repairs.is_empty()
1245            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246        {
1247            let claim = FixClaim::acquire(&self.state.dir())?;
1248            let disk = RunState::load(&self.state.id)?;
1249            let checkpoint = |s: &RunState| {
1250                (
1251                    s.merge_repairs.len(),
1252                    s.merge_repairs
1253                        .last()
1254                        .map(|r| (r.question.clone(), r.phase)),
1255                    s.land_approval
1256                        .as_ref()
1257                        .map(|a| (a.question.clone(), a.head.clone())),
1258                )
1259            };
1260            if checkpoint(&disk) != checkpoint(&self.state) {
1261                bail!(
1262                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263                );
1264            }
1265            Some(claim)
1266        } else {
1267            None
1268        };
1269        let mut execution = self.claim_driver().await?;
1270        let runner = &mut *execution.runner;
1271        let wait = runner.repository_wait();
1272        // Keep the graph's large future off the callers' stack, including
1273        // standalone task attempts running on a default-sized test thread.
1274        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275        let result = match result {
1276            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277                record_repository_block(&mut runner.state, &e);
1278                runner.park_here()?;
1279                Ok(())
1280            }
1281            other => other,
1282        };
1283        execution.driver.finish(&mut runner.state).await?;
1284        let ended = if result.is_err() {
1285            Some({
1286                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287                if let Err(error) = &result {
1288                    notice.message = format!(
1289                        "Run {} stopped with an error: {}",
1290                        runner.state.id,
1291                        error.to_string().lines().next().unwrap_or("unknown error")
1292                    );
1293                }
1294                notice
1295            })
1296        } else {
1297            crate::notices::run_ended(&runner.state)
1298        };
1299        if let Some(notice) = ended {
1300            crate::notices::raise(notice);
1301        }
1302        result
1303    }
1304
1305    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307    }
1308
1309    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310        let wait = self.repository_wait();
1311        coordinate_operation(&mut self.state, cwd, &wait).await
1312    }
1313
1314    /// Validate ownership and persist the driver claim in one critical section.
1315    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316        let owner = match crate::pr_head::owner_lock_wait(
1317            &crate::run::home(),
1318            &self.state.id,
1319            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320        )
1321        .await
1322        {
1323            Ok(owner) => owner,
1324            Err(error) => {
1325                let detail = format!("worktree maintenance or ownership update: {error:#}");
1326                // Admission did not own run.json: retain this resource outcome
1327                // in memory for queue settlement without overwriting a concurrent
1328                // retirement or publication transaction.
1329                self.state.resource_block = Some(crate::run::ResourceBlock {
1330                    cause: crate::run::ResourceBlockCause::Other,
1331                    node: "start".into(),
1332                    kind: "worktree maintenance".into(),
1333                    detail: detail.clone(),
1334                    at: Timestamp::now(),
1335                });
1336                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337            }
1338        };
1339        let disk = RunState::load(&self.state.id)?;
1340        crate::pr_head::ensure_owner(&disk)?;
1341        let claimed =
1342            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343        // Only proven-live ownership refuses here. Unknown records (legacy runs
1344        // with no claim marker or process identity) fall through to the
1345        // exclusive execution lease taken just below, which is what actually
1346        // keeps a second driver out.
1347        anyhow::ensure!(
1348            disk.liveness(claimed) != crate::run::Liveness::Live,
1349            "run already has a live driver; wait for its execution to end"
1350        );
1351        if let Some(to) = &disk.released_to {
1352            bail!(
1353                "run {} cannot continue: its worktree was released to run {}",
1354                self.state.short(),
1355                crate::run::short_of(to)
1356            );
1357        }
1358        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359        let mut execution = DriverExecution {
1360            runner: self,
1361            driver,
1362        };
1363        // Declare after the execution guard: on any claim error this short
1364        // lock must drop before that guard reloads and releases ownership.
1365        let _owner = owner;
1366        let state = &mut execution.runner.state;
1367        // Moving again, so it is no longer parked. Set before the walk rather
1368        // than in `resume`, so every way of re-entering the graph clears it
1369        // and a card cannot claim a run is waiting to be resumed while the
1370        // agents are already working.
1371        state.parked = false;
1372        state.resource_wait = None;
1373        state.resource_block = None;
1374        // Any seat this state still lists as answering belongs to whatever
1375        // process last drove this run — this one included, if it crashed
1376        // mid-wave. Cleared and flushed immediately, before anything else
1377        // runs, so a resume can never show a seat as live when nothing is
1378        // asking it anything yet; the node that actually dispatches the next
1379        // wave repopulates it.
1380        state.clear_active();
1381        // Recorded in the same spot, and flushed together with the clear
1382        // above: this is the pid a reader checks (`RunState::liveness`) when
1383        // no daemon claim exists to answer "is a process still driving this
1384        // run" — a plain `magi run` / `magi review` typed into a terminal
1385        // claims nothing there. Always overwritten, never only-if-absent, so
1386        // a resumed run's stale pid from a previous, possibly-dead process
1387        // can never survive into this one's own report. Unlike
1388        // `clear_active`, this changes on every single `execute()` call, so
1389        // the save below is now unconditional rather than only-if-cleared.
1390        //
1391        // `driver_started_at` is recorded in the same breath, from this same
1392        // pid, so `liveness` can tell a live pid that is genuinely still us
1393        // apart from one the OS has since handed to an unrelated process —
1394        // see that field's own doc for why the pid alone is not enough.
1395        let pid = std::process::id();
1396        state.driver_pid = Some(pid);
1397        state.driver_started_at = crate::proc::process_started_at(pid);
1398        state.driver_exited = false;
1399        state.driver_claimed = Some(true);
1400        state.begin_pass();
1401        execution.driver.bind(state);
1402        state.save()?;
1403        execution.driver.record(state)?;
1404        Ok(execution)
1405    }
1406
1407    async fn execute_graph(&mut self) -> Result<()> {
1408        self.state.configure_recovery_resume()?;
1409        // Preservation may invalidate an old empty classification. Resolve it
1410        // before any resume shortcut can reuse a verdict or publish a branch.
1411        self.persist_candidates().await?;
1412        if self
1413            .state
1414            .merge_decision
1415            .as_ref()
1416            .is_some_and(|decision| !decision.closed)
1417            && self.state.status != RunStatus::Landing
1418        {
1419            if crate::merge_decision::resumable_hold(&self.state) {
1420                self.state.merge = Some(MergeOutcome {
1421                    mode: MergeMode::Pr,
1422                    ok: false,
1423                    detail: self
1424                        .state
1425                        .merge_decision
1426                        .as_ref()
1427                        .unwrap()
1428                        .target
1429                        .pr
1430                        .clone(),
1431                    empty: false,
1432                });
1433                self.run_land().await?;
1434                self.settle_questions();
1435                return Ok(());
1436            }
1437            bail!(
1438                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439            );
1440        }
1441        if self.state.config.graph.selected_implementer.is_some()
1442            && self.state.status == RunStatus::Blocked
1443            && self.state.tally.is_none()
1444            && self.state.candidate_preservation.is_empty()
1445            && self.state.interrupted_implementations.is_empty()
1446        {
1447            bail!(
1448                "selected implementation is held; explicitly release the owning task to retry it"
1449            );
1450        }
1451
1452        if self.state.merge_repairs.last().is_some_and(|r| {
1453            matches!(
1454                r.phase,
1455                crate::run::MergeRepairPhase::Requested
1456                    | crate::run::MergeRepairPhase::Fixing
1457                    | crate::run::MergeRepairPhase::Reviewing
1458            )
1459        }) {
1460            if self.repair_merge_review().await? {
1461                self.run_land().await?;
1462            }
1463            self.settle_questions();
1464            return Ok(());
1465        }
1466        // Publication already passed review and gate. Resume only that node:
1467        // syncing to a newer base here could change the verified commit.
1468        if self.state.pr_publication_pending {
1469            self.merge().await?;
1470            if self.resource_halt()? {
1471                return Ok(());
1472            }
1473            self.state.save()?;
1474            self.settle_questions();
1475            return Ok(());
1476        }
1477        // A run that already lost its quorum never resumes into the verdict
1478        // machinery: `deliberate` and `vote` would otherwise clobber the
1479        // stalled marker back to Voting and the run would keep going past a
1480        // verdict that is no longer trustworthy. Everything already recorded is
1481        // kept, so the run stays resumable (or foldable) for a human to pick up.
1482        //
1483        // On --resume the run gets one chance to repair itself: the seats a
1484        // rate limit took out are re-asked. If their quota has since reset and
1485        // the quorum is restored, the run picks up and finishes; otherwise it
1486        // stays stale and still-resumable for a later retry. If it does not
1487        // recover, the returned status stays `Stalled` and nothing was
1488        // clobbered (the recovery only mutates entries for the lost seats).
1489        if self.state.status == RunStatus::Stalled {
1490            if self.recover_stall().await? {
1491                self.finish_after_tally().await?;
1492            } else {
1493                // Still below quorum: persist the marker and stay resumable.
1494                self.state.save()?;
1495            }
1496            return Ok(());
1497        }
1498        // A run parked inside `land` - watching CI, mid fix-round, or
1499        // waiting on the owner's merge approval - resumes directly into it,
1500        // never back through `prep`. Everything before `merge` already
1501        // concluded; that is the only way `status` reaches `Landing` in the
1502        // first place. Re-walking `review_loop` first would also be actively
1503        // wrong: its own status recomputation (see its doc) treats any
1504        // clean round as reason to set `status` to `Gating`, which would
1505        // clobber this marker before `merge` ever ran, and this run would
1506        // never find its way back into `land` at all.
1507        if self.state.status == RunStatus::Landing
1508            || crate::handover::landing_checkpoint(&self.state)
1509        {
1510            self.state.land_sync_pending = true;
1511            self.state.save()?;
1512            self.run_land().await?;
1513            // `run_land` may have settled the run right here - CI came back
1514            // green and the PR merged, say - without ever passing back
1515            // through `merge`'s own trailing call. Whatever it left `status`
1516            // as is what this has to read.
1517            self.settle_questions();
1518            return Ok(());
1519        }
1520        // A run parked at `merge` on the owner's word about a withheld PR text
1521        // resumes straight into `merge`. Re-walking the tail would sync to the
1522        // base again (possibly rebasing, possibly spending the fixer) after the
1523        // gate had passed, and then push a tree nobody reviewed or gated.
1524        if self.state.github_text.is_some()
1525            && self.state.merge.is_none()
1526            && self.state.status == RunStatus::Gating
1527        {
1528            self.merge().await?;
1529            if self.resource_halt()? {
1530                return Ok(());
1531            }
1532            self.state.save()?;
1533            return Ok(());
1534        }
1535        if let Err(e) = self.prep().await {
1536            if e.is::<crate::repo_lock::ResourceBlocked>() {
1537                return Err(e);
1538            }
1539            self.state.status = RunStatus::Blocked;
1540            self.state
1541                .prep_failure
1542                .get_or_insert_with(|| format!("{e:#}"));
1543            self.state
1544                .event("prep", format!("preparation stopped: {e:#}"));
1545            self.state.save()?;
1546            return Err(e);
1547        }
1548        if self.park_here()? {
1549            return Ok(());
1550        }
1551        self.advise().await?;
1552        if self.park_here()? {
1553            return Ok(());
1554        }
1555        self.implement().await?;
1556        if self.park_here()? {
1557            return Ok(());
1558        }
1559        // `after_implement` already saved the state and settled any open
1560        // questions when it set this; nothing later in the graph has
1561        // anything to judge.
1562        if self.state.status == RunStatus::VerifiedNoop {
1563            return Ok(());
1564        }
1565        self.judge().await?;
1566        if self.park_here()? {
1567            return Ok(());
1568        }
1569        self.deliberate().await?;
1570        if self.park_here()? {
1571            return Ok(());
1572        }
1573        self.vote().await?;
1574        if self.park_here()? {
1575            return Ok(());
1576        }
1577        self.tally()?;
1578        // A verdict that lost its quorum is not trustworthy: do not review,
1579        // gate, or merge on it. Everything already done is kept, so the run
1580        // stays resumable (or foldable); the human can replace the agent that
1581        // ran out of quota and pick it up.
1582        if self.state.status == RunStatus::Stalled {
1583            // Persist the stalled marker now — the normal end-of-execute save
1584            // below is below this early return, and without it a resumed run
1585            // would reload a pre-tally status and keep going.
1586            self.state.save()?;
1587            return Ok(());
1588        }
1589        self.finish_after_tally().await?;
1590        Ok(())
1591    }
1592
1593    /// A wave could not start its agents because a shared resource stayed busy
1594    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1595    /// resumable. Unstarted seats have no result; completed peers retain
1596    /// their saved candidate state.
1597    fn stop_for_resource(&mut self) -> Result<()> {
1598        // Resource acquisition observes cancellation inside a node. Complete
1599        // the same durable park as every other node boundary before returning.
1600        if self.pause.parked() || self.interrupt.parked() {
1601            self.park_here()?;
1602            return Ok(());
1603        }
1604        if self.state.resource_block.is_some() {
1605            self.state.status = RunStatus::Blocked;
1606        }
1607        self.state.save()
1608    }
1609
1610    /// Stop walking the graph if a node ended on a resource block.
1611    fn resource_halt(&mut self) -> Result<bool> {
1612        if self.state.resource_block.is_none() {
1613            return Ok(false);
1614        }
1615        self.stop_for_resource()?;
1616        Ok(true)
1617    }
1618
1619    /// Park here if asked to, recording it in the run's own timeline.
1620    ///
1621    /// Returns whether the caller should stop walking the graph. The state is
1622    /// saved either way by the node that just finished; this adds the event so
1623    /// the operator's card says why a run that is neither finished nor moving
1624    /// is sitting where it is.
1625    fn park_here(&mut self) -> Result<bool> {
1626        let parking = self.pause.parked() || self.interrupt.parked();
1627        if !parking && self.state.resource_block.is_some() {
1628            self.stop_for_resource()?;
1629            return Ok(true);
1630        }
1631        // Either handle asking is enough - see `Pause`'s own doc for why
1632        // they are never the same one. `interrupt` is checked second so a
1633        // reason it carries is preferred in the message below over a plain
1634        // shutdown park racing it at the same boundary.
1635        if !self.pause.parked() && !self.interrupt.parked() {
1636            return Ok(false);
1637        }
1638        if self.state.parked {
1639            return Ok(true);
1640        }
1641        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642        if let Some(block) = &mut self.state.resource_block
1643            && matches!(
1644                block.cause,
1645                crate::run::ResourceBlockCause::Cancelled { .. }
1646            )
1647        {
1648            block.cause = crate::run::ResourceBlockCause::Cancelled {
1649                park: cause.clone(),
1650            };
1651        }
1652        let why = match cause.as_ref().map(ParkCause::label) {
1653            Some(reason) => format!(
1654                "parked after `{}` ({reason}) — resume to carry on from here",
1655                self.state.status.as_str()
1656            ),
1657            None => format!(
1658                "parked after `{}` — resume to carry on from here",
1659                self.state.status.as_str()
1660            ),
1661        };
1662        self.state.event("park", why);
1663        self.state
1664            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665        self.state.parked = true;
1666        self.state.save()?;
1667        Ok(true)
1668    }
1669
1670    /// Only a new explicit operator approval may retry failed preparation.
1671    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672        if self.state.prep_failure.is_none() {
1673            return Ok(());
1674        }
1675        let failed_at = self
1676            .state
1677            .events
1678            .iter()
1679            .rev()
1680            .find(|event| event.node == "prep")
1681            .map_or(self.state.updated_at, |event| event.at);
1682        anyhow::ensure!(
1683            approval_at > failed_at,
1684            "preparation retry requires a new explicit release after the failure"
1685        );
1686        self.state.prep_failure = None;
1687        self.state.prep_complete = Some(false);
1688        self.state
1689            .event("prep-retry", "operator approved retry of preparation");
1690        Ok(())
1691    }
1692
1693    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1694    pub fn on_pause(&mut self, pause: Pause) {
1695        self.pause = pause;
1696    }
1697
1698    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1699    /// scheduler asking this one run - and no other - to park so a task
1700    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1701    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1702    /// handle.
1703    pub fn watch_interrupt(&mut self, pause: Pause) {
1704        self.interrupt = pause;
1705    }
1706
1707    /// Abandon this run's own open questions, once `status` has actually
1708    /// settled rather than merely paused.
1709    ///
1710    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1711    /// either back up with the candidates, the review round and the seat
1712    /// sessions already on disk, so a question an implementer asked mid-round
1713    /// may still get a real answer read by a real resume. Only the statuses
1714    /// `resumable` excludes are actually final: the run merged, it reached
1715    /// `Ready` with nothing left to do, it failed outright with no
1716    /// established point to continue from, or every candidate agreed, with
1717    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1718    /// every one of those the seat that asked is gone for good, exactly like
1719    /// the run being deleted under `magi run rm` - so the same cleanup
1720    /// applies, worded for what actually happened instead of "the run was
1721    /// deleted".
1722    ///
1723    /// Best-effort and silent on success: called from every place `status`
1724    /// can land on one of those three, including ones a resumed run revisits,
1725    /// so it must cost nothing when there was nothing open to begin with.
1726    fn settle_questions(&mut self) {
1727        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729        }
1730    }
1731
1732    /// The tail of the graph after a trustworthy tally: fold losers, review,
1733    /// gate, merge, and persist.
1734    async fn finish_after_tally(&mut self) -> Result<()> {
1735        // Before the losers are folded: their branches are what the graft
1736        // reads from.
1737        self.graft().await?;
1738        if self.resource_halt()? {
1739            return Ok(());
1740        }
1741        self.fold_losers().await?;
1742        // Before review starts, and again right before the gate: a run's
1743        // review rounds can themselves take long enough for the base to move
1744        // a second time, and the gate is the one node whose "green" gets
1745        // acted on.
1746        self.sync_to_base().await?;
1747        if self.resource_halt()? {
1748            return Ok(());
1749        }
1750        if self.state.status == RunStatus::AlreadyInBase {
1751            return Ok(());
1752        }
1753        self.review_loop().await?;
1754        if self.resource_halt()? {
1755            return Ok(());
1756        }
1757        self.sync_to_base().await?;
1758        if self.resource_halt()? {
1759            return Ok(());
1760        }
1761        if self.state.status == RunStatus::AlreadyInBase {
1762            return Ok(());
1763        }
1764        self.gate().await?;
1765        self.merge().await?;
1766        if self.resource_halt()? {
1767            return Ok(());
1768        }
1769        self.state.save()?;
1770        Ok(())
1771    }
1772
1773    // ---------------------------------------------------------------- prep
1774
1775    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1776    /// run with the step and its output in the event; it is never swallowed.
1777    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1778    /// resume starts over instead of finding candidates and missing seats.
1779    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780        let result = match ensure_setup_config(&mut self.state, repo).await {
1781            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782            Err(e) => Err(e),
1783        };
1784        if let Err(e) = result {
1785            git::worktree_remove_clean(repo, wt).await.ok();
1786            if in_prep {
1787                self.rollback_prep(repo).await;
1788            }
1789            self.state.status = RunStatus::Blocked;
1790            self.state.event(
1791                "setup",
1792                format!("worktree setup failed in {}: {e:#}", wt.display()),
1793            );
1794            self.state.save()?;
1795            return Err(e);
1796        }
1797        Ok(())
1798    }
1799
1800    /// Undo everything `prep` created, so a resume re-runs it whole.
1801    async fn rollback_prep(&mut self, repo: &Path) {
1802        let root = self.state.worktree_root();
1803        for c in std::mem::take(&mut self.state.candidates) {
1804            if matches!(
1805                git::worktree_remove_clean(repo, &c.worktree).await,
1806                Ok(true)
1807            ) {
1808                git::branch_delete(repo, &c.branch).await.ok();
1809            } else {
1810                self.state.candidates.push(c);
1811            }
1812        }
1813        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814        for j in 1..=seats {
1815            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816                .await
1817                .ok();
1818        }
1819        for k in 1..=self.state.config.graph.advisors {
1820            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821                .await
1822                .ok();
1823        }
1824    }
1825
1826    async fn prep(&mut self) -> Result<()> {
1827        if let Some(reason) = &self.state.prep_failure {
1828            bail!(
1829                "candidate preparation previously failed: {reason}; correct the source before retrying"
1830            );
1831        }
1832        if self
1833            .state
1834            .prep_complete
1835            .unwrap_or(!self.state.candidates.is_empty())
1836        {
1837            return Ok(());
1838        }
1839        let repo = self.state.repo.clone();
1840        // Includes rollback, config, refs, checkout creation and setup. No
1841        // repository mutation may precede the real shared lock acquisition.
1842        let _coordination = self.coordinate(&repo).await?;
1843        let base = self.state.base_commit.clone();
1844        let plan = self.state.sources.plan(&repo, &base).await?;
1845        self.rollback_prep(&repo).await;
1846        if !self.state.candidates.is_empty() {
1847            bail!(
1848                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849            );
1850        }
1851        self.state.prep_complete = Some(false);
1852        self.state.status = RunStatus::Prep;
1853        self.state.save()?;
1854        let repo = self.state.repo.clone();
1855        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856        let root = self.state.worktree_root();
1857        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859        // The hook is the write-time half of the blindness contract; the
1860        // presentation filter in `blind` is the half that cannot be bypassed.
1861        let hooks_dir = self.state.dir().join("hooks");
1862        if self.state.config.blind.commit_msg_hook {
1863            std::fs::create_dir_all(&hooks_dir)
1864                .with_context(|| format!("create {}", hooks_dir.display()))?;
1865            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866            let path = hooks_dir.join("commit-msg");
1867            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868            make_executable(&path)?;
1869            // Ref-counted rather than a plain idempotent set: with more than
1870            // one run able to be in flight in the same repository at once
1871            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1872            // true?" check cannot tell "another run of mine still needs
1873            // this" from "nobody does", and the run that happens to finish
1874            // first would disable the hook out from under a sibling still
1875            // relying on it.
1876            git::acquire_worktree_config(&repo).await?;
1877            self.state.enabled_worktree_config = true;
1878        }
1879
1880        // `[worktree] setup` hides its products per worktree, which needs the
1881        // same per-worktree config the hook does. Held until fold, like it.
1882        ensure_setup_config(&mut self.state, &repo).await?;
1883
1884        for (index, (spec, label)) in self
1885            .roles
1886            .implementers
1887            .clone()
1888            .into_iter()
1889            .zip(labels)
1890            .enumerate()
1891        {
1892            let branch = self.state.branch_for(label);
1893            let worktree = root.join(format!("cand-{label}"));
1894            // Persist intent before Git can leave a partially created tree.
1895            // Candidate registration follows setup, so it cannot inventory
1896            // crashes or refusals during creation itself.
1897            self.state.event(
1898                "prep-intent",
1899                serde_json::json!({
1900                    "branch": branch,
1901                    "worktree": worktree,
1902                    "start": start,
1903                    "picks": plan.picks,
1904                })
1905                .to_string(),
1906            );
1907            self.state.save()?;
1908            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909            if self.state.config.blind.commit_msg_hook {
1910                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911            }
1912            git::local_exclude(&worktree, "/.magi/").await?;
1913            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914                // Nothing of this prep may survive: a resume must start over,
1915                // not find half the candidates and skip the rest.
1916                git::worktree_remove_clean(&repo, &worktree).await.ok();
1917                git::branch_delete(&repo, &branch).await.ok();
1918                self.rollback_prep(&repo).await;
1919                self.state.status = RunStatus::Blocked;
1920                self.state
1921                    .event("prep", format!("worktree setup failed: {e:#}"));
1922                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923                self.state.save()?;
1924                return Err(e);
1925            }
1926            self.state.candidates.push(Candidate {
1927                index,
1928                label,
1929                agent: spec.id.clone(),
1930                branch,
1931                worktree,
1932                summary: String::new(),
1933                stat: String::new(),
1934                files: 0,
1935                commits: 0,
1936                empty: false,
1937                failed: None,
1938                verified_noop: None,
1939                duration_ms: 0,
1940                folded: false,
1941            });
1942            self.state.save()?;
1943            for pick in &plan.picks {
1944                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945                {
1946                    self.state.status = RunStatus::Blocked;
1947                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948                    self.state
1949                        .event("prep", format!("cannot apply referenced commit: {e}"));
1950                    self.state.save()?;
1951                    return Err(e);
1952                }
1953            }
1954        }
1955
1956        for j in 1..=self.roles.judges.len() {
1957            let wt = root.join(format!("judge-{j}"));
1958            if !wt.exists() {
1959                git::worktree_add_detached(&repo, &wt, &base).await?;
1960                self.setup_seat_worktree(&repo, &wt, true).await?;
1961            }
1962        }
1963
1964        // Disposable, detached checkouts for the design-deliberation stage's
1965        // advisor seats — the same shape as the judges' above, at the same
1966        // base commit, since advisors also only ever read. Sized off the
1967        // configured count directly rather than a resolved roster: unlike
1968        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1969        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1970        // `prep` has no `ResolvedRoles` field to read a count from here.
1971        if self.state.config.graph.advise {
1972            for k in 1..=self.state.config.graph.advisors {
1973                let wt = root.join(format!("advisor-{k}"));
1974                if !wt.exists() {
1975                    git::worktree_add_detached(&repo, &wt, &base).await?;
1976                    self.setup_seat_worktree(&repo, &wt, true).await?;
1977                }
1978            }
1979        }
1980
1981        // A judge cannot tell it is looking at its own patch — the seats keep
1982        // separate conversations — but a panel that shares agents with the
1983        // field is less independent than it looks, and that is worth saying out
1984        // loud once per run rather than leaving it in the config.
1985        let authors: Vec<&str> = self
1986            .roles
1987            .implementers
1988            .iter()
1989            .map(|a| a.id.as_str())
1990            .collect();
1991        let overlap: Vec<String> = self
1992            .roles
1993            .judges
1994            .iter()
1995            .enumerate()
1996            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998            .collect();
1999        if !overlap.is_empty() {
2000            let note = format!(
2001                "{} also authored a candidate; blind, but the panel is less \
2002                 independent than {} distinct agents would be",
2003                overlap.join(", "),
2004                self.roles.judges.len()
2005            );
2006            self.state.event("prep", note);
2007        }
2008
2009        self.state.event(
2010            "prep",
2011            format!(
2012                "{} candidates, {} judges, base {} ({})",
2013                self.state.candidates.len(),
2014                self.roles.judges.len(),
2015                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016                self.state.base_branch
2017            ),
2018        );
2019        self.state.prep_complete = Some(true);
2020        self.state.status = RunStatus::Implementing;
2021        self.state.save()?;
2022        Ok(())
2023    }
2024
2025    // -------------------------------------------------------------- advise
2026
2027    /// The design-deliberation stage: independent, read-only advisor seats
2028    /// each sketch a design before any implementer touches the repository,
2029    /// and (when at least one produced a usable proposal) a synthesis seat
2030    /// blends them into a brief `implement` carries in every candidate's
2031    /// prompt.
2032    ///
2033    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2034    /// advisors` is the proposal count. Everything here is best-effort and
2035    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2036    /// that cannot reach quota, or a synthesis seat that produced nothing
2037    /// usable all leave `implement` exactly as it was before this stage
2038    /// existed — the task instruction alone — rather than failing the whole
2039    /// competition over an enrichment stage. Every outcome is still recorded
2040    /// as an event, so a run that got nothing from this stage says why.
2041    ///
2042    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2043    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2044    /// resumed run whose stage failed would re-run it, and re-spend the
2045    /// agent calls, on every reentry before `implement`.
2046    ///
2047    /// Also skipped once any candidate shows implementation progress — the
2048    /// exact predicate `implement` itself uses to decide a candidate is no
2049    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2050    /// is not enough: a run created by an older binary that predates this
2051    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2052    /// an already-`Implementing`-or-later run under this build would
2053    /// otherwise walk straight back through `prep` (a no-op once candidates
2054    /// exist) into this node and spawn every advisor seat against worktrees
2055    /// `prep` never recreated — after implementation has already started,
2056    /// which is exactly the invariant this stage exists to guarantee.
2057    async fn advise(&mut self) -> Result<()> {
2058        let implement_untouched = self
2059            .state
2060            .candidates
2061            .iter()
2062            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063        if !self.state.config.graph.advise || self.state.advise_attempted {
2064            return Ok(());
2065        }
2066        if !implement_untouched {
2067            self.state.event(
2068                "advise",
2069                "skipping the design-deliberation stage: at least one \
2070                 candidate already shows implementation progress, so this \
2071                 run is past the point the stage exists to run before"
2072                    .to_owned(),
2073            );
2074            self.state.advise_attempted = true;
2075            self.state.save()?;
2076            return Ok(());
2077        }
2078        let run_id = self.state.id.clone();
2079        let prompts = self.state.config.prompts.clone();
2080        let instruction = self.state.instruction.clone();
2081        let language = self.state.config.graph.language.clone();
2082        let root = self.state.worktree_root();
2083        let n = self.state.config.graph.advisors;
2084        let where_recorded = self.state.dir().join("run.json");
2085
2086        let seats = match self.state.config.advisors() {
2087            Ok(seats) if !seats.is_empty() => seats,
2088            Ok(_) => {
2089                self.state.event(
2090                    "advise",
2091                    format!(
2092                        "[graph] advisors is 0; skipping the design-deliberation \
2093                         stage and continuing without a synthesis brief (see {})",
2094                        where_recorded.display()
2095                    ),
2096                );
2097                self.state.advise_attempted = true;
2098                self.state.save()?;
2099                return Ok(());
2100            }
2101            Err(e) => {
2102                self.state.event(
2103                    "advise",
2104                    format!(
2105                        "could not resolve advisor seats ({e:#}); continuing \
2106                         without a design-deliberation brief (see {})",
2107                        where_recorded.display()
2108                    ),
2109                );
2110                self.state.advise_attempted = true;
2111                self.state.save()?;
2112                return Ok(());
2113            }
2114        };
2115
2116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117        let artifacts = agent::artifacts_dir(&self.state.dir());
2118        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120        let mut jobs = Vec::new();
2121        for (i, spec) in seats.iter().cloned().enumerate() {
2122            let seat_key = format!("advisor-{}", i + 1);
2123            let seat = self.seat(&seat_key, &spec.id);
2124            jobs.push(SeatJob {
2125                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126                spec,
2127                seat,
2128                cwd: worktrees[i % worktrees.len()].clone(),
2129                timeout,
2130                allow_write: false,
2131                sessions: false,
2132                artifacts: artifacts.clone(),
2133                stem: seat_key,
2134                handover: None,
2135            });
2136        }
2137
2138        self.state.event(
2139            "advise",
2140            format!(
2141                "{} advisor seat(s) sketching a design in parallel",
2142                jobs.len()
2143            ),
2144        );
2145        let mut quota_losses = Vec::new();
2146        let cache = self.state.config.cache_dir();
2147        let ctx = WaveCtx {
2148            carry_seats: false,
2149            pauses: Some((&self.pause, &self.interrupt)),
2150            run: &run_id,
2151            node: "advise",
2152            prompts: &prompts,
2153            cache: cache.as_deref(),
2154            round: None,
2155        };
2156        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157        let results = ask_json_wave::<Proposal>(
2158            jobs,
2159            Arc::clone(&self.sem),
2160            self.state.config.graph.retries,
2161            &advisor_roster,
2162            &ctx,
2163            &mut quota_losses,
2164            &mut self.state,
2165            &|p: &Proposal| p.validate(),
2166        )
2167        .await;
2168        if self.state.resource_block.is_some() {
2169            return self.stop_for_resource();
2170        }
2171        self.state.quota.extend(quota_losses);
2172
2173        let mut records = Vec::with_capacity(results.len());
2174        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175            let agent_id = seat.agent.clone();
2176            self.state.seats.insert(seat.key.clone(), seat);
2177            match res {
2178                Ok((proposal, out)) => {
2179                    self.state
2180                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2181                    records.push(advise::AdvisorRecord::proposed(
2182                        i + 1,
2183                        agent_id,
2184                        proposal,
2185                        out.duration_ms,
2186                    ));
2187                }
2188                Err(e) => {
2189                    self.state.event(
2190                        "advise",
2191                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192                    );
2193                    records.push(advise::AdvisorRecord::failed(
2194                        i + 1,
2195                        agent_id,
2196                        e.to_string(),
2197                    ));
2198                }
2199            }
2200        }
2201
2202        let mut advice = advise::Advice {
2203            records,
2204            synthesis: None,
2205        };
2206        if advice.proposals().is_empty() {
2207            self.state.event(
2208                "advise",
2209                "no advisor produced a usable proposal; continuing without a \
2210                 synthesis brief"
2211                    .to_owned(),
2212            );
2213        } else {
2214            match self
2215                .synthesize_brief(
2216                    &advice,
2217                    &instruction,
2218                    &language,
2219                    &worktrees[0],
2220                    &artifacts,
2221                    &run_id,
2222                    &prompts,
2223                    cache.as_deref(),
2224                )
2225                .await
2226            {
2227                Ok(Some(text)) => {
2228                    self.state.event(
2229                        "advise",
2230                        "synthesized a design brief for the implementer".to_owned(),
2231                    );
2232                    advice.synthesis = Some(text);
2233                }
2234                Ok(None) => {
2235                    self.state.event(
2236                        "advise",
2237                        "the synthesis seat produced nothing usable; continuing \
2238                         without a design brief"
2239                            .to_owned(),
2240                    );
2241                }
2242                Err(e) => {
2243                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2244                        return Err(e);
2245                    }
2246                    self.state.event(
2247                        "advise",
2248                        format!("could not synthesize a design brief: {e:#}"),
2249                    );
2250                }
2251            }
2252        }
2253        advise::apply_reflection(&mut advice);
2254
2255        self.state.advice = Some(advice);
2256        self.state.advise_attempted = true;
2257        self.state.save()?;
2258        Ok(())
2259    }
2260
2261    /// The synthesis seat: reads every advisor's proposal and blends them
2262    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2263    /// out of [`Runner::advise`] only for readability — it is not called
2264    /// anywhere else.
2265    ///
2266    /// Picked the same way [`crate::talk`]'s standing conversation and
2267    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2268    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2269    /// order (a claude seat, else the first runnable agent in roster order)
2270    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2271    /// in [`crate::config`] for why a dedicated field exists here at all.
2272    #[allow(clippy::too_many_arguments)]
2273    async fn synthesize_brief(
2274        &mut self,
2275        advice: &advise::Advice,
2276        instruction: &str,
2277        language: &str,
2278        cwd: &Path,
2279        artifacts: &Path,
2280        run_id: &str,
2281        prompts: &Prompts,
2282        cache: Option<&Path>,
2283    ) -> Result<Option<String>> {
2284        let chain = agent::pick_chain(
2285            &self.state.config.agents,
2286            self.state.config.roles.synthesizer.as_ref(),
2287            &agent::installed,
2288            "synthesizer",
2289        )?;
2290        let proposals = advice.proposals();
2291        let mut prompt = prompt::with_overlay(
2292            prompt::synthesize_brief(instruction, &proposals, language),
2293            prompts.overlay("advise"),
2294        );
2295        if cache.is_some() {
2296            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2297            // below — see `prompt::build_cache_note`'s doc for why telling a
2298            // read-only seat to build through the shared cache is exactly how
2299            // a sandbox's write refusal gets misread as a defect.
2300            prompt.push('\n');
2301            prompt.push_str(&prompt::build_cache_note("advise", false));
2302        }
2303        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304        // Each id is tried once, in order; a quota hit, error or unusable
2305        // answer moves to the next. The seat is single-turn (`sessions:
2306        // false`) and the prompt is the whole context, so a fallback agent
2307        // needs nothing carried over.
2308        let mut last = None;
2309        for (n, spec) in chain.iter().enumerate() {
2310            if n > 0 {
2311                self.state
2312                    .event("advise", format!("synthesis falling back to {}", spec.id));
2313            }
2314            let mut seat = self.seat("advise-synthesis", &spec.id);
2315            let wait = self.repository_wait();
2316            let outcome = wait
2317                .scope(agent::invoke(
2318                    spec,
2319                    &mut seat,
2320                    &Invocation {
2321                        coordination: self.state.config.coordination.as_ref(),
2322                        fetched_base: None,
2323                        cwd,
2324                        prompt: &prompt,
2325                        timeout,
2326                        allow_write: false,
2327                        sessions: false,
2328                        artifacts,
2329                        stem: &if n == 0 {
2330                            "advise-synthesis".to_owned()
2331                        } else {
2332                            format!("advise-synthesis-{}", spec.id)
2333                        },
2334                        run: run_id,
2335                        node: "advise",
2336                        cache_dir: None,
2337                        attachments: &[],
2338                        writable: &[],
2339                    },
2340                ))
2341                .await;
2342            if outcome.is_ok() {
2343                self.state.seats.insert(seat.key.clone(), seat);
2344            }
2345            let advance = agent::chain_advances(&outcome);
2346            last = Some(outcome);
2347            if !advance {
2348                break;
2349            }
2350        }
2351        // Exhausted: the last attempt's result is what a single failed seat
2352        // would have produced.
2353        let out = last.expect("a chain holds at least one agent")?;
2354        if !out.usable() {
2355            return Ok(None);
2356        }
2357        let text =
2358            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2359        Ok((!text.trim().is_empty()).then_some(text))
2360    }
2361
2362    // ----------------------------------------------------------- implement
2363
2364    async fn implement(&mut self) -> Result<()> {
2365        self.persist_candidates().await?;
2366        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2367        // agent files with `magi task add` name the run that paid for it. The
2368        // prompt overlay is cloned alongside it because the waves borrow it
2369        // while `self` is mutably borrowed by the node's own bookkeeping.
2370        let run_id = self.state.id.clone();
2371        let prompts = self.state.config.prompts.clone();
2372        let todo: Vec<usize> = self
2373            .state
2374            .candidates
2375            .iter()
2376            .enumerate()
2377            .filter(|(_, c)| {
2378                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2379                    && c.failed.is_none()
2380                    && !c.empty
2381            })
2382            .map(|(i, _)| i)
2383            .collect();
2384        if todo.is_empty() {
2385            return self.after_implement();
2386        }
2387        self.state.status = RunStatus::Implementing;
2388
2389        let language = self.state.config.graph.language.clone();
2390        let github_language = self.state.config.graph.github_language.clone();
2391        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2392        let sessions = self.state.config.graph.sessions;
2393        let artifacts = agent::artifacts_dir(&self.state.dir());
2394        // The design-deliberation stage's blended brief, when `advise` found
2395        // one — carried into every implementer's prompt the same way
2396        // regardless of which candidate it is.
2397        let brief = self
2398            .state
2399            .advice
2400            .as_ref()
2401            .and_then(|a| a.synthesis.as_deref())
2402            .map(str::to_owned);
2403        let attachments = self.state.attachments.clone();
2404
2405        let mut jobs = Vec::new();
2406        for &i in &todo {
2407            let (index, label, worktree) = {
2408                let c = &self.state.candidates[i];
2409                (c.index, c.label, c.worktree.clone())
2410            };
2411            let spec = self.roles.implementers[index].clone();
2412            let seat_key = format!("impl-{label}");
2413            let seat = self.seat(&seat_key, &spec.id);
2414            let instruction = seeded_instruction(&self.state);
2415            jobs.push(SeatJob {
2416                spec,
2417                seat,
2418                prompt: prompt::implement_in(
2419                    &instruction,
2420                    &worktree.to_string_lossy(),
2421                    &language,
2422                    &github_language,
2423                    brief.as_deref(),
2424                    &attachments,
2425                ),
2426                cwd: worktree,
2427                timeout,
2428                allow_write: true,
2429                sessions,
2430                artifacts: artifacts.clone(),
2431                stem: format!("impl-{label}"),
2432                handover: None,
2433            });
2434        }
2435
2436        self.state.event(
2437            "implement",
2438            format!("{} candidates in parallel", jobs.len()),
2439        );
2440        // Kept so a seat whose CLI hung up can be asked again from the same
2441        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2442        // can update a seat's own entry once a fallback agent takes it over —
2443        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2444        // whichever agent actually answered, not the one that quota'd out.
2445        let mut sent = jobs.clone();
2446        let cache = self.state.config.cache_dir();
2447        let ctx = WaveCtx {
2448            carry_seats: false,
2449            pauses: Some((&self.pause, &self.interrupt)),
2450            run: &run_id,
2451            node: "implement",
2452            prompts: &prompts,
2453            cache: cache.as_deref(),
2454            round: None,
2455        };
2456        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2457        if !wave_blocked(&results) {
2458            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2459                .await;
2460            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2461                .await?;
2462            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2463                .await;
2464        }
2465        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2466
2467        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2468            && results
2469                .iter()
2470                .any(|(_, _, out)| FailClass::of(out).is_some());
2471        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2472            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2473                // Executed turns interrupted by disk pressure already carry
2474                // a preservation barrier from wave. Admission-only waits do
2475                // not. Keep the interrupted seat's session for recovery too.
2476                if self
2477                    .state
2478                    .candidate_preservation
2479                    .contains_key(&self.state.candidates[i].label)
2480                {
2481                    self.state.seats.insert(seat.key.clone(), seat);
2482                }
2483                continue;
2484            }
2485            let seat_key = seat.key.clone();
2486            // A quota fallback (`resume_seat_handovers`) may have handed this
2487            // seat to a different agent than the one `prep` recorded on the
2488            // candidate; the stats tables and any later fixer-defaults-to-
2489            // winner's-author lookup must credit whoever actually answered —
2490            // unless every fallback also quota'd out, in which case nobody
2491            // actually answered and crediting the last agent tried would
2492            // erase every earlier agent's own quota loss from the stats
2493            // tables instead of just this one seat's.
2494            let agent = seat.agent.clone();
2495            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2496            self.state.seats.insert(seat.key.clone(), seat);
2497            let label = self.state.candidates[i].label;
2498            self.state.interrupted_implementations.remove(&label);
2499
2500            let (summary, duration, failed, verified_claim) = match out {
2501                AgentOutcome::Ok(o) => {
2502                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2503                    let failed = (!o.usable()).then(|| {
2504                        if o.timed_out {
2505                            "agent timed out".to_owned()
2506                        } else {
2507                            format!(
2508                                "agent exited with {}",
2509                                crate::notices::exit_status(o.exit_code)
2510                            )
2511                        }
2512                    });
2513                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2514                    (text, o.duration_ms, failed, verified_claim)
2515                }
2516                // Left un-resumed by `resume_undelivered` (a dirty tree
2517                // already rescues the work, or there was no session left to
2518                // resume into) — reported like the ordinary failure it is,
2519                // never as if `o.text` (the CLI's raw error JSON) were an
2520                // answer.
2521                AgentOutcome::Dropped(o) => {
2522                    let why = o
2523                        .dropped
2524                        .as_ref()
2525                        .map(|d| d.why.as_str())
2526                        .unwrap_or("the CLI ended the stream without delivering its answer");
2527                    (
2528                        String::new(),
2529                        o.duration_ms,
2530                        Some(format!("the CLI dropped the stream ({why})")),
2531                        None,
2532                    )
2533                }
2534                AgentOutcome::Quota(o) => {
2535                    self.state.quota.push(QuotaLoss {
2536                        seat: seat_key,
2537                        node: "implement".to_owned(),
2538                        at: Timestamp::now(),
2539                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2540                    });
2541                    (
2542                        String::new(),
2543                        o.duration_ms,
2544                        Some("rate limited (quota); produced no change".to_owned()),
2545                        None,
2546                    )
2547                }
2548                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2549                    (String::new(), 0, Some(e), None)
2550                }
2551            };
2552
2553            // Save every completed turn before any fallible Git operation. A failed
2554            // rescue must not erase the reply or prevent collection of other seats.
2555            let c = &mut self.state.candidates[i];
2556            if !exhausted_the_fallback_chain {
2557                c.agent = agent;
2558            }
2559            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2560            c.duration_ms = duration;
2561            c.failed = failed;
2562            c.verified_noop = verified_claim;
2563            self.state
2564                .candidate_preservation
2565                .insert(label, "awaiting persistence".to_owned());
2566        }
2567        self.state.save()?;
2568        self.persist_candidates().await?;
2569
2570        if resource_blocked {
2571            return self.stop_for_resource();
2572        }
2573        if selected_failed {
2574            self.state.status = RunStatus::Blocked;
2575            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2576            self.state.save()?;
2577            bail!(
2578                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2579            );
2580        }
2581        self.after_implement()
2582    }
2583
2584    /// Retry only persistence after a completed implementation turn.
2585    async fn persist_candidates(&mut self) -> Result<()> {
2586        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2587        for label in labels {
2588            let i = self
2589                .state
2590                .candidates
2591                .iter()
2592                .position(|c| c.label == label)
2593                .context("pending candidate is missing")?;
2594            let c = self.state.candidates[i].clone();
2595            let interrupted = self.state.interrupted_implementations.contains(&label);
2596            let result = async {
2597                let wait = crate::repo_lock::WaitPolicy::new(
2598                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2599                    || false,
2600                );
2601                let _coordination =
2602                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2603                // Best effort evidence requires no index writes. A successful snapshot
2604                // never authorizes removal of a tree whose rescue failed.
2605                if let Err(e) = git::snapshot_uncommitted(
2606                    &c.worktree,
2607                    &self
2608                        .state
2609                        .dir()
2610                        .join("artifacts")
2611                        .join(format!("candidate-{label}-recovery")),
2612                )
2613                .await
2614                {
2615                    self.state.event(
2616                        "preserve",
2617                        format!("candidate {label} snapshot incomplete: {e:#}"),
2618                    );
2619                }
2620                let rescued = git::rescue_commit(
2621                    &c.worktree,
2622                    &format!("magi: candidate {label} (uncommitted work)"),
2623                )
2624                .await?;
2625                self.state.note_withheld("implement", &rescued.withheld);
2626                let base = &self.state.base_commit;
2627                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2628                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2629                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2630                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2631                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2632                if c.empty && commits > 0 && !patch.trim().is_empty()
2633                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2634                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2635                        || self.state.tally.is_some())
2636                {
2637                    // Keep the old classification and pending marker so every
2638                    // retry stops too. Retain decisions as recovery evidence;
2639                    // only a supported recovery operation may replace them.
2640                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2641                }
2642                let c = &mut self.state.candidates[i];
2643                c.commits = commits;
2644                c.stat = stat;
2645                c.files = files;
2646                let no_changes = commits == 0 || patch.trim().is_empty();
2647                c.empty = no_changes && !interrupted;
2648                if interrupted && no_changes {
2649                    c.failed = None;
2650                    c.verified_noop = None;
2651                }
2652                if !c.empty {
2653                    c.failed = None;
2654                    c.verified_noop = None;
2655                }
2656                self.state.event(
2657                    "preserve",
2658                    format!("candidate {label}: {files} files, {commits} commits"),
2659                );
2660                Ok::<_, anyhow::Error>(())
2661            }
2662            .await;
2663            match result {
2664                Ok(()) => {
2665                    self.state.candidate_preservation.remove(&label);
2666                    if !interrupted || self.state.candidates[i].files > 0 {
2667                        self.state.interrupted_implementations.remove(&label);
2668                    }
2669                }
2670                Err(e) => {
2671                    let reason = format!("candidate {label} preservation failed: {e:#}");
2672                    self.state.event("preserve", &reason);
2673                    self.state.candidate_preservation.insert(label, reason);
2674                }
2675            }
2676            if self.state.candidate_preservation.is_empty()
2677                && self.state.status == RunStatus::Blocked
2678                && self.state.candidates.iter().all(|c| c.failed.is_none())
2679            {
2680                // Persist the recovered phase with the cleared pending marker;
2681                // a crash between saves must not turn this into a selected-seat
2682                // failure hold on the next resume.
2683                self.state.status = RunStatus::Implementing;
2684                self.state.event(
2685                    "preserve",
2686                    "candidate persistence recovered; implementation can continue",
2687                );
2688            }
2689            self.state.save()?;
2690        }
2691        if !self.state.candidate_preservation.is_empty() {
2692            self.state.status = RunStatus::Blocked;
2693            self.state.save()?;
2694            bail!(
2695                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2696            );
2697        }
2698        Ok(())
2699    }
2700
2701    /// Ask again, once, for work a CLI did and then failed to hand over.
2702    ///
2703    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2704    /// status with an empty response and a usage report showing output tokens,
2705    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2706    /// seven minutes and 14,267 output tokens that arrived as an empty
2707    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2708    ///
2709    /// Two conditions, and both matter:
2710    ///
2711    /// - **Only when the tree is untouched.** Often the agent has already
2712    ///   written its files and only the closing message was lost; the rescue
2713    ///   commit below picks that up and there is nothing to ask for. Re-asking
2714    ///   then would pay for a second implementation of work already on disk.
2715    /// - **Once.** A CLI that drops one stream can drop the next, and this
2716    ///   node is the most expensive in the graph.
2717    ///
2718    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2719    /// dropped reply still carried its `conversation_id`, so the seat is asked
2720    /// to finish what it was doing rather than sent the whole task again. It
2721    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2722    /// node's - for the same reason a re-ranked judge does: restating finished
2723    /// work is not the work.
2724    ///
2725    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2726    /// same way until it resets, while an abandoned conversation is still
2727    /// there to be picked up.
2728    async fn resume_undelivered(
2729        &mut self,
2730        results: &mut [(usize, SeatState, AgentOutcome)],
2731        sent: &[SeatJob],
2732        prompts: &Prompts,
2733        run_id: &str,
2734    ) {
2735        for (wi, seat, out) in results.iter_mut() {
2736            let Some(dropped) = (match &*out {
2737                AgentOutcome::Dropped(o) => o.dropped.clone(),
2738                _ => None,
2739            }) else {
2740                continue;
2741            };
2742            let Some(job) = sent.get(*wi) else { continue };
2743            // Already on disk? Then only the closing message was lost.
2744            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2745                self.state.event(
2746                    "implement",
2747                    format!(
2748                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2749                         work is in the tree",
2750                        seat.key, dropped.output_tokens, dropped.why
2751                    ),
2752                );
2753                continue;
2754            }
2755            // The re-ask only makes sense as a resume: `resume_after_drop`
2756            // says nothing about the task, trusting the seat to still hold it.
2757            // Without a session to resume — sessions disabled, or this CLI's
2758            // drop shape happened not to carry a session id — that prompt
2759            // would open a brand-new conversation with no context at all,
2760            // which is worse than leaving this as the ordinary failure it
2761            // already is.
2762            if !has_context(&job.spec, seat, job.sessions) {
2763                self.state.event(
2764                    "implement",
2765                    format!(
2766                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2767                         is no session left to resume",
2768                        seat.key, dropped.output_tokens, dropped.why
2769                    ),
2770                );
2771                continue;
2772            }
2773            self.state.event(
2774                "implement",
2775                format!(
2776                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2777                     conversation",
2778                    seat.key, dropped.output_tokens, dropped.why
2779                ),
2780            );
2781            let mut retry = job.clone();
2782            retry.seat = seat.clone();
2783            retry.prompt = prompt::resume_after_drop(&dropped.why);
2784            retry.timeout = retry_budget(job.timeout, true);
2785            retry.stem = format!("{}-resume", job.stem);
2786            let cache = self.state.config.cache_dir();
2787            let ctx = WaveCtx {
2788                carry_seats: false,
2789                pauses: Some((&self.pause, &self.interrupt)),
2790                run: run_id,
2791                node: "implement",
2792                prompts,
2793                cache: cache.as_deref(),
2794                round: None,
2795            };
2796            let (resumed_seat, resumed) =
2797                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2798            *seat = resumed_seat;
2799            *out = resumed;
2800        }
2801    }
2802
2803    /// Fall an implement seat through to the next untried agent in the
2804    /// implementer roster when it lost to quota — or, since the handover was
2805    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2806    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2807    /// quota path itself being unchanged — instead of leaving the
2808    /// seat's loss final the moment one agent's account runs dry.
2809    ///
2810    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2811    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2812    /// truncates to the single slot rotation picked, so a solo task whose one
2813    /// implementer hits quota mid-run used to have nothing else to try. This
2814    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2815    /// unrotated roster — which is the only place the *other* candidates in
2816    /// the machine's roster still exist once `implementers` has been cut down
2817    /// to size.
2818    ///
2819    /// Walks forward from just past the seat's own original position in the
2820    /// roster, never wrapping back to the front: a later candidate slot (say
2821    /// `beta`, the roster's second entry) must fall through to the *next*
2822    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2823    /// almost certainly a different candidate's own agent already — and once
2824    /// the roster's tail is exhausted there is nothing left to fall through
2825    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2826    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2827    /// let this retry that id forever. The loop keeps falling through until
2828    /// an attempt lands something other than `Quota` or the roster's tail
2829    /// runs out of untried ids, at which point the seat is left exactly as
2830    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2831    /// `QuotaLoss` recorded, the candidate failed/empty.
2832    ///
2833    /// `sent` is taken mutably and updated with the fallback agent's spec:
2834    /// `resume_unconfirmed_commands`, which runs after this and also reads
2835    /// `sent`, must see whichever agent actually ended up answering the seat
2836    /// — reading the stale, original spec there would check session
2837    /// eligibility against the wrong CLI and could hand a fallback agent's
2838    /// session id to the agent that just lost the seat to quota.
2839    ///
2840    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2841    /// — `self.seat` only reuses state when the agent id is unchanged, so
2842    /// handing it a different id already gets this for free. Reusing the old
2843    /// seat would resume a different CLI's session as if it were a
2844    /// continuation of this one.
2845    ///
2846    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2847    /// a quota loss cuts an agent off mid-turn, so anything already in the
2848    /// tree is unfinished work, not a completed candidate a re-ask would pay
2849    /// for twice. A dirty tree is rescued into a commit first (the same
2850    /// neutral-identity rescue `implement`'s own outcome loop gives every
2851    /// candidate) so the next agent starts clean.
2852    ///
2853    /// The new agent gets the implementer's full prompt and full
2854    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2855    /// it has no session and no context, and is implementing the task from
2856    /// nothing, unlike a resumed drop which is only restating work already
2857    /// done.
2858    ///
2859    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2860    /// `implement` event, never into `self.state.quota` — that is what
2861    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2862    /// go unspent, and a seat that ultimately recovered on its second or
2863    /// third agent is not the stalled panel that check exists to catch. Only
2864    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2865    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2866    /// outcome loop already has — this helper never pushes to it itself.
2867    async fn resume_seat_handovers(
2868        &mut self,
2869        results: &mut [(usize, SeatState, AgentOutcome)],
2870        sent: &mut [SeatJob],
2871        prompts: &Prompts,
2872        run_id: &str,
2873    ) -> Result<()> {
2874        let instruction = seeded_instruction(&self.state);
2875        let language = self.state.config.graph.language.clone();
2876        let github_language = self.state.config.graph.github_language.clone();
2877        let brief = self
2878            .state
2879            .advice
2880            .as_ref()
2881            .and_then(|a| a.synthesis.as_deref())
2882            .map(str::to_owned);
2883        let attachments = self.state.attachments.clone();
2884        for (wi, seat, out) in results.iter_mut() {
2885            // Who holds the other candidate seats of this wave right now
2886            // (earlier handovers already written back to `sent`).
2887            let others: BTreeSet<String> = sent
2888                .iter()
2889                .enumerate()
2890                .filter(|(j, _)| j != wi)
2891                .map(|(_, j)| j.spec.id.clone())
2892                .collect();
2893            let Some(job) = sent.get_mut(*wi) else {
2894                continue;
2895            };
2896            // Where the seat's own original agent sits in the roster — the
2897            // fallback walk starts just past here, never at the front, so a
2898            // later candidate slot's quota loss does not fall back onto an
2899            // earlier slot's own agent.
2900            let start = self
2901                .roles
2902                .implementer_roster
2903                .iter()
2904                .position(|s| s.id == job.spec.id)
2905                .unwrap_or(0);
2906            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2907            let mut fallback_attempt = 0usize;
2908            let mut prev: Option<FailClass> = None;
2909            while let Some(cur) = FailClass::of(&*out) {
2910                if !should_hand_over(prev.as_ref(), &cur) {
2911                    break;
2912                }
2913                let Some(next) =
2914                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2915                        .cloned()
2916                else {
2917                    break;
2918                };
2919                tried.insert(next.id.clone());
2920                fallback_attempt += 1;
2921
2922                let coordination = self.coordinate(&job.cwd).await?;
2923                if let Ok(r) = git::rescue_commit(
2924                    &job.cwd,
2925                    &format!(
2926                        "magi: candidate {} (uncommitted work before {} fallback)",
2927                        seat.key,
2928                        if cur == FailClass::Quota {
2929                            "quota"
2930                        } else {
2931                            "handover"
2932                        }
2933                    ),
2934                )
2935                .await
2936                {
2937                    self.state.note_withheld("implement", &r.withheld);
2938                }
2939                drop(coordination);
2940
2941                record_handover(
2942                    &mut self.state,
2943                    "implement",
2944                    &seat.key,
2945                    &seat.agent,
2946                    &next.id,
2947                    &cur,
2948                    &fail_reason(&*out),
2949                );
2950                prev = Some(cur.clone());
2951
2952                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2953                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2954                // Kept in sync on `sent` itself, not just the local retry: a
2955                // later helper (`resume_unconfirmed_commands`) reads `sent`
2956                // after this one returns and must see whichever agent is now
2957                // occupying the seat, not the one that just quota'd out —
2958                // otherwise it would judge session/continuation eligibility
2959                // by the wrong CLI and could resend a fallback's session id
2960                // to the agent that lost it the seat in the first place.
2961                job.spec = next.clone();
2962                let mut retry = job.clone();
2963                retry.seat = new_seat;
2964                retry.prompt = prompt::implement_in(
2965                    &instruction,
2966                    &job.cwd.to_string_lossy(),
2967                    &language,
2968                    &github_language,
2969                    brief.as_deref(),
2970                    &attachments,
2971                );
2972                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2973                let cache = self.state.config.cache_dir();
2974                let ctx = WaveCtx {
2975                    carry_seats: false,
2976                    pauses: Some((&self.pause, &self.interrupt)),
2977                    run: run_id,
2978                    node: "implement",
2979                    prompts,
2980                    cache: cache.as_deref(),
2981                    round: None,
2982                };
2983                let (fallback_seat, fallback_out) = run_one(
2984                    retry,
2985                    Arc::clone(&self.sem),
2986                    &ctx,
2987                    &mut self.state,
2988                    fallback_attempt,
2989                )
2990                .await;
2991                *seat = fallback_seat;
2992                *out = fallback_out;
2993            }
2994        }
2995        Ok(())
2996    }
2997
2998    /// Ask an implement seat's own CLI to confirm what it started, once, when
2999    /// its reply reported a command whose completion status it never
3000    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3001    /// that does and does not mean.
3002    ///
3003    /// The completion contract this task asks for, extended to `implement`
3004    /// with the same signal `continue_fix_report` reads for the fixer,
3005    /// rather than a keyword search over the reply or a hard requirement on
3006    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3007    /// a candidate's CLI turn ended cleanly while a test run it had started
3008    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3009    /// named in it at all is untouched by this: `commands` is empty, so
3010    /// there is nothing to be unconfirmed.
3011    ///
3012    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3013    /// this is not about recovering edits that might already be on disk, it
3014    /// is about a result the seat itself never vouched for, which resuming
3015    /// asks for regardless of what the tree already holds. Bounded to one
3016    /// attempt for the same reason `resume_undelivered` is — this is the
3017    /// most expensive node in the graph — and a seat that still cannot
3018    /// confirm on that attempt is left as whatever its (possibly still
3019    /// unconfirmed) reply says; this does not invent a new "failed" reason
3020    /// for a candidate that otherwise produced a real, committed change.
3021    async fn resume_unconfirmed_commands(
3022        &mut self,
3023        results: &mut [(usize, SeatState, AgentOutcome)],
3024        sent: &[SeatJob],
3025        prompts: &Prompts,
3026        run_id: &str,
3027    ) {
3028        for (wi, seat, out) in results.iter_mut() {
3029            let AgentOutcome::Ok(o) = &*out else {
3030                continue;
3031            };
3032            if !has_unconfirmed_command(&o.commands) {
3033                continue;
3034            }
3035            let Some(job) = sent.get(*wi) else { continue };
3036            if !has_context(&job.spec, seat, job.sessions) {
3037                self.state.event(
3038                    "implement",
3039                    format!(
3040                        "{}: the reply named a command whose own CLI never confirmed the exit \
3041                         status of, but there is no session left to resume",
3042                        seat.key
3043                    ),
3044                );
3045                continue;
3046            }
3047            self.state.event(
3048                "implement",
3049                format!(
3050                    "{}: the reply named a command whose own CLI never confirmed the exit \
3051                     status of; resuming the conversation",
3052                    seat.key
3053                ),
3054            );
3055            let mut retry = job.clone();
3056            retry.seat = seat.clone();
3057            retry.prompt = prompt::resume_incomplete(
3058                "a command in your last reply had no confirmed exit status",
3059            );
3060            retry.timeout = retry_budget(job.timeout, true);
3061            retry.stem = format!("{}-confirm", job.stem);
3062            let cache = self.state.config.cache_dir();
3063            let ctx = WaveCtx {
3064                carry_seats: false,
3065                pauses: Some((&self.pause, &self.interrupt)),
3066                run: run_id,
3067                node: "implement",
3068                prompts,
3069                cache: cache.as_deref(),
3070                round: None,
3071            };
3072            let (resumed_seat, resumed) =
3073                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3074            *seat = resumed_seat;
3075            *out = resumed;
3076        }
3077    }
3078
3079    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3080    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3081    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3082    /// that motivated this.
3083    ///
3084    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3085    /// own nudge loop already covers for judge/review/vote seats, and not a
3086    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3087    /// implement seats: here the CLI turn genuinely finished while the node's
3088    /// own work — the fixer's account of what it did — had not. Gated purely
3089    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3090    /// reply, never on any wording in it, so a fixer whose valid, first-try
3091    /// `FixReport` happens to mention having waited on a background test is
3092    /// never resumed — the `Ok(report)` branch at the call site returns
3093    /// before this is ever invoked.
3094    ///
3095    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3096    /// attempt ([`retry_budget`]), nothing attempted once the session is
3097    /// gone, and a quota hit ends the loop immediately rather than retrying a
3098    /// rate limit that fails the same way again.
3099    async fn continue_fix_report(
3100        &mut self,
3101        mut seat: SeatState,
3102        parse_err: String,
3103        job: &SeatJob,
3104        prompts: &Prompts,
3105        run_id: &str,
3106        round: usize,
3107    ) -> (
3108        SeatState,
3109        Option<FixReport>,
3110        Option<String>,
3111        ContinuationRecord,
3112    ) {
3113        let mut last_err = parse_err;
3114        let mut cumulative_wait_ms = 0u64;
3115        let mut attempts = 0usize;
3116        loop {
3117            if !has_context(&job.spec, &seat, job.sessions) {
3118                self.state.event(
3119                    "fix",
3120                    format!(
3121                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3122                         session left to resume into"
3123                    ),
3124                );
3125                let outcome = if attempts == 0 {
3126                    ContinuationOutcome::NoSession
3127                } else {
3128                    ContinuationOutcome::Exhausted
3129                };
3130                return (
3131                    seat,
3132                    None,
3133                    Some(format!("unparsable fix report: {last_err}")),
3134                    ContinuationRecord {
3135                        attempts,
3136                        cumulative_wait_ms,
3137                        outcome,
3138                    },
3139                );
3140            }
3141            if attempts >= MAX_FIX_CONTINUATIONS {
3142                self.state.event(
3143                    "fix",
3144                    format!(
3145                        "round {round}: fixer's reply still had no adoption report after \
3146                         {attempts} continuation(s) ({last_err}); giving up"
3147                    ),
3148                );
3149                return (
3150                    seat,
3151                    None,
3152                    Some(format!(
3153                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3154                    )),
3155                    ContinuationRecord {
3156                        attempts,
3157                        cumulative_wait_ms,
3158                        outcome: ContinuationOutcome::Exhausted,
3159                    },
3160                );
3161            }
3162            attempts += 1;
3163            self.state.event(
3164                "fix",
3165                format!(
3166                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3167                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3168                ),
3169            );
3170            let mut retry = job.clone();
3171            retry.seat = seat.clone();
3172            retry.prompt = prompt::resume_incomplete(&last_err);
3173            retry.timeout = retry_budget(job.timeout, true);
3174            retry.stem = format!("{}-continue{attempts}", job.stem);
3175            let cache = self.state.config.cache_dir();
3176            let ctx = WaveCtx {
3177                carry_seats: false,
3178                pauses: Some((&self.pause, &self.interrupt)),
3179                run: run_id,
3180                node: "fix",
3181                prompts,
3182                cache: cache.as_deref(),
3183                round: Some(round),
3184            };
3185            let (resumed_seat, resumed_out) = run_one(
3186                retry,
3187                Arc::clone(&self.sem),
3188                &ctx,
3189                &mut self.state,
3190                attempts,
3191            )
3192            .await;
3193            seat = resumed_seat;
3194            match resumed_out {
3195                AgentOutcome::Ok(o) => {
3196                    cumulative_wait_ms += o.duration_ms;
3197                    match verdict::extract_json::<FixReport>(&o.text) {
3198                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3199                            self.state.event(
3200                                "fix",
3201                                format!(
3202                                    "round {round}: fixer's adoption report recovered after \
3203                                     {attempts} continuation(s)"
3204                                ),
3205                            );
3206                            return (
3207                                seat,
3208                                Some(report),
3209                                None,
3210                                ContinuationRecord {
3211                                    attempts,
3212                                    cumulative_wait_ms,
3213                                    outcome: ContinuationOutcome::Resumed,
3214                                },
3215                            );
3216                        }
3217                        // The report parsed, but this same reply's own
3218                        // CommandEvidence — the identical record `state.jobs`
3219                        // renders — names a command whose CLI never
3220                        // confirmed an exit status. Read together, that is
3221                        // not a resolved answer: keep nudging rather than
3222                        // accept a report standing next to a command the
3223                        // seat's own CLI cannot vouch for.
3224                        Ok(_) => {
3225                            last_err = "the reply parsed, but it reported a command whose own CLI \
3226                                 never confirmed an exit status"
3227                                .to_owned();
3228                        }
3229                        Err(e) => last_err = e.to_string(),
3230                    }
3231                }
3232                AgentOutcome::Quota(o) => {
3233                    cumulative_wait_ms += o.duration_ms;
3234                    self.state.quota.push(QuotaLoss {
3235                        seat: seat.key.clone(),
3236                        node: "fix".to_owned(),
3237                        at: Timestamp::now(),
3238                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3239                    });
3240                    self.state.event(
3241                        "fix",
3242                        format!(
3243                            "round {round}: continuation rate limited (quota); not retrying now"
3244                        ),
3245                    );
3246                    return (
3247                        seat,
3248                        None,
3249                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3250                        ContinuationRecord {
3251                            attempts,
3252                            cumulative_wait_ms,
3253                            outcome: ContinuationOutcome::QuotaLost,
3254                        },
3255                    );
3256                }
3257                AgentOutcome::Dropped(o) => {
3258                    cumulative_wait_ms += o.duration_ms;
3259                    let why = o
3260                        .dropped
3261                        .as_ref()
3262                        .map(|d| d.why.as_str())
3263                        .unwrap_or("the CLI ended the stream without delivering its answer");
3264                    last_err = format!("the CLI dropped the stream ({why})");
3265                }
3266                AgentOutcome::Failed(e) => last_err = e,
3267                // The continuation was never started; the report stays lost
3268                // and the caller records that, without any handover.
3269                AgentOutcome::ResourceBlocked(e) => {
3270                    return (
3271                        seat,
3272                        None,
3273                        Some(format!("the continuation was not started: {e}")),
3274                        ContinuationRecord {
3275                            attempts: attempts.saturating_sub(1),
3276                            cumulative_wait_ms,
3277                            outcome: ContinuationOutcome::NoSession,
3278                        },
3279                    );
3280                }
3281            }
3282        }
3283    }
3284
3285    fn after_implement(&mut self) -> Result<()> {
3286        if !self.state.candidate_preservation.is_empty() {
3287            bail!("candidate preservation incomplete; judging is unsafe");
3288        }
3289        // Scan every candidate patch once the set is complete.
3290        if self.state.leaks.is_empty() {
3291            let cfg = self.state.config.blind.clone();
3292            let mut leaks = Vec::new();
3293            for c in &self.state.candidates {
3294                let Some(patch) =
3295                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3296                else {
3297                    continue;
3298                };
3299                leaks.extend(blind::scan(
3300                    &format!("candidate {} patch", c.label),
3301                    &patch,
3302                    &cfg.vendor_tokens,
3303                ));
3304            }
3305            if !leaks.is_empty() {
3306                let summary = leaks
3307                    .iter()
3308                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3309                    .collect::<Vec<_>>()
3310                    .join(", ");
3311                match cfg.on_leak {
3312                    LeakPolicy::Fail => {
3313                        self.state.status = RunStatus::Failed;
3314                        self.state
3315                            .event("blind", format!("vendor text in a patch: {summary}"));
3316                        self.state.leaks = leaks;
3317                        self.state.save()?;
3318                        self.settle_questions();
3319                        bail!(
3320                            "blind.on_leak = \"fail\" and vendor text reached a \
3321                             judged patch: {summary}"
3322                        );
3323                    }
3324                    LeakPolicy::Redact => self.state.event(
3325                        "blind",
3326                        format!("redacting vendor text for judging: {summary}"),
3327                    ),
3328                    LeakPolicy::Warn => self.state.event(
3329                        "blind",
3330                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3331                    ),
3332                }
3333                self.state.leaks = leaks;
3334            }
3335        }
3336
3337        if self.state.viable().is_empty() {
3338            if self.state.all_candidates_verified_noop() {
3339                // Every candidate agreed, with evidence the adoption guard
3340                // accepted, that nothing belongs in this worktree. That is
3341                // not the same fact as a candidate that simply failed to
3342                // write anything, and settling it as an ordinary `Failed`
3343                // (see `SCHEMA`'s doc for schema 10) is what let two of
3344                // task 391f's attempts burn a retry each re-discovering the
3345                // same already-landed fix. Terminal either way, so `judge`
3346                // must never run over an empty candidate set — unlike the
3347                // `Failed` branch below this returns `Ok`, not an error:
3348                // nothing here failed.
3349                self.state.status = RunStatus::VerifiedNoop;
3350                self.state.save()?;
3351                self.settle_questions();
3352                return Ok(());
3353            }
3354            self.state.status = RunStatus::Failed;
3355            self.state.save()?;
3356            self.settle_questions();
3357            bail!("no candidate produced a change; nothing to judge");
3358        }
3359        self.state.status = RunStatus::Judging;
3360        self.state.save()?;
3361        Ok(())
3362    }
3363
3364    // --------------------------------------------------------------- judge
3365
3366    async fn judge(&mut self) -> Result<()> {
3367        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3368        // agent files with `magi task add` name the run that paid for it. The
3369        // prompt overlay is cloned alongside it because the waves borrow it
3370        // while `self` is mutably borrowed by the node's own bookkeeping.
3371        let run_id = self.state.id.clone();
3372        let prompts = self.state.config.prompts.clone();
3373        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3374            return Ok(());
3375        }
3376        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3377        if viable.len() == 1 {
3378            // Recorded so this is a one-time event: `judgements` stays empty
3379            // either way, which without this flag is indistinguishable from
3380            // "not yet judged" on the next reentry — and status is left
3381            // untouched, so a later node's conclusion (e.g. `Blocked` after
3382            // the review budget ran out) survives a resume instead of being
3383            // clobbered back to `Judging` by this node running again.
3384            self.state.judge_skipped = true;
3385            self.state.event(
3386                "judge",
3387                format!(
3388                    "only candidate {} produced a change; judging skipped",
3389                    viable[0].label
3390                ),
3391            );
3392            self.state.save()?;
3393            return Ok(());
3394        }
3395        self.state.status = RunStatus::Judging;
3396
3397        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3398        let language = self.state.config.graph.language.clone();
3399        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3400        let sessions = self.state.config.graph.sessions;
3401        let artifacts = agent::artifacts_dir(&self.state.dir());
3402        let root = self.state.worktree_root();
3403        let base_short = short(&self.state.base_commit);
3404
3405        let mut jobs = Vec::new();
3406        let mut orders = Vec::new();
3407        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3408            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3409            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3410            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3411            let seat_key = format!("judge-{}", j + 1);
3412            let seat = self.seat(&seat_key, &spec.id);
3413            jobs.push(SeatJob {
3414                prompt: prompt::judge(
3415                    &self.state.instruction,
3416                    &views,
3417                    self.roles.judges.len(),
3418                    &base_short,
3419                    &language,
3420                ),
3421                spec,
3422                seat,
3423                cwd: root.join(format!("judge-{}", j + 1)),
3424                timeout,
3425                allow_write: false,
3426                sessions,
3427                artifacts: artifacts.clone(),
3428                stem: format!("judge-{}", j + 1),
3429                handover: None,
3430            });
3431        }
3432
3433        self.state.event(
3434            "judge",
3435            format!(
3436                "{} judges ranking {} candidates blind",
3437                jobs.len(),
3438                viable.len()
3439            ),
3440        );
3441        let labels_for_check = labels.clone();
3442        let mut quota_losses = Vec::new();
3443        let cache = self.state.config.cache_dir();
3444        let ctx = WaveCtx {
3445            carry_seats: false,
3446            pauses: Some((&self.pause, &self.interrupt)),
3447            run: &run_id,
3448            node: "judge",
3449            prompts: &prompts,
3450            cache: cache.as_deref(),
3451            round: None,
3452        };
3453        let results = ask_json_wave::<Ranking>(
3454            jobs,
3455            Arc::clone(&self.sem),
3456            self.state.config.graph.retries,
3457            &self.roles.judge_roster,
3458            &ctx,
3459            &mut quota_losses,
3460            &mut self.state,
3461            &move |r: &Ranking| r.validate(&labels_for_check),
3462        )
3463        .await;
3464        if self.state.resource_block.is_some() {
3465            return self.stop_for_resource();
3466        }
3467        self.state.quota.extend(quota_losses);
3468
3469        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3470            let agent_id = seat.agent.clone();
3471            self.state.seats.insert(seat.key.clone(), seat);
3472            let mut record = Judgement {
3473                judge: j + 1,
3474                seat: format!("judge-{}", j + 1),
3475                agent: agent_id,
3476                ranking: Vec::new(),
3477                reasons: BTreeMap::new(),
3478                confidence: None,
3479                strengths: BTreeMap::new(),
3480                order: orders[j].clone(),
3481                failed: None,
3482                duration_ms: 0,
3483            };
3484            match res {
3485                Ok((ranking, out)) => {
3486                    record.ranking = ranking.normalized();
3487                    record.reasons = ranking.reasons;
3488                    record.strengths = ranking.strengths;
3489                    record.confidence = ranking.confidence;
3490                    record.duration_ms = out.duration_ms;
3491                    self.state.event(
3492                        "judge",
3493                        format!(
3494                            "judge {} ranked {}",
3495                            j + 1,
3496                            record.ranking.iter().collect::<String>()
3497                        ),
3498                    );
3499                }
3500                Err(e) => {
3501                    record.failed = Some(e.to_string());
3502                    self.state
3503                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3504                }
3505            }
3506            self.state.judgements.push(record);
3507            self.state.save()?;
3508        }
3509        Ok(())
3510    }
3511
3512    // ---------------------------------------------------------- deliberate
3513
3514    async fn deliberate(&mut self) -> Result<()> {
3515        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3516        // agent files with `magi task add` name the run that paid for it. The
3517        // prompt overlay is cloned alongside it because the waves borrow it
3518        // while `self` is mutably borrowed by the node's own bookkeeping.
3519        let run_id = self.state.id.clone();
3520        let prompts = self.state.config.prompts.clone();
3521        if !self.state.deliberation.is_empty() {
3522            return Ok(());
3523        }
3524        let tops: Vec<char> = self
3525            .state
3526            .judgements
3527            .iter()
3528            .filter_map(|j| j.ranking.first().copied())
3529            .collect();
3530        let rounds = self.state.config.graph.deliberate_rounds;
3531        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3532            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3533                self.state.event(
3534                    "deliberate",
3535                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3536                );
3537            }
3538            self.state.status = RunStatus::Voting;
3539            self.state.save()?;
3540            return Ok(());
3541        }
3542
3543        self.state.status = RunStatus::Deliberating;
3544        self.state.event(
3545            "deliberate",
3546            format!(
3547                "split: first choices were {} — opening {rounds} round(s)",
3548                tops.iter().collect::<String>()
3549            ),
3550        );
3551
3552        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3553        let language = self.state.config.graph.language.clone();
3554        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3555        let sessions = self.state.config.graph.sessions;
3556        let artifacts = agent::artifacts_dir(&self.state.dir());
3557        let root = self.state.worktree_root();
3558        let base_short = short(&self.state.base_commit);
3559
3560        // Judges argue in sequence so that a turn can answer the one before it;
3561        // that is the difference between deliberation and three parallel
3562        // monologues.
3563        for round in 1..=rounds {
3564            let mut turns: Vec<DeliberationTurn> = Vec::new();
3565            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3566                if self.state.judgements[j].failed.is_some() {
3567                    continue;
3568                }
3569                let seat_key = format!("judge-{}", j + 1);
3570                let spec = self.occupant(&seat_key, spec);
3571                let mut seat = self.seat(&seat_key, &spec.id);
3572                let transcript = self.transcript(&turns, j);
3573                let build = |context: Option<&str>| {
3574                    prompt::deliberate(
3575                        &self.state.instruction,
3576                        context,
3577                        &transcript,
3578                        round,
3579                        rounds,
3580                        &language,
3581                    )
3582                };
3583                let block = self.candidate_block(&viable, &base_short);
3584                let full = build(Some(&block));
3585                let text = if has_context(&spec, &seat, sessions) {
3586                    build(None)
3587                } else {
3588                    full.clone()
3589                };
3590                let job = SeatJob {
3591                    spec,
3592                    seat: seat.clone(),
3593                    prompt: text,
3594                    cwd: root.join(format!("judge-{}", j + 1)),
3595                    timeout,
3596                    allow_write: false,
3597                    sessions,
3598                    artifacts: artifacts.clone(),
3599                    stem: format!("delib-{round}-judge-{}", j + 1),
3600                    handover: Some(full),
3601                };
3602                let cache = self.state.config.cache_dir();
3603                let ctx = WaveCtx {
3604                    carry_seats: false,
3605                    pauses: Some((&self.pause, &self.interrupt)),
3606                    run: &run_id,
3607                    node: "deliberate",
3608                    prompts: &prompts,
3609                    cache: cache.as_deref(),
3610                    round: None,
3611                };
3612                // A turn is never nudged (`retries` 0): a failed seat is
3613                // handed to the next roster agent, which gets the full
3614                // context. An empty answer is a turn, not a failure.
3615                let mut losses = Vec::new();
3616                let mut results = ask_wave_with::<String>(
3617                    vec![job],
3618                    Arc::clone(&self.sem),
3619                    0,
3620                    &self.roles.judge_roster,
3621                    &ctx,
3622                    &mut losses,
3623                    &mut self.state,
3624                    &|text: &str| {
3625                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3626                    },
3627                )
3628                .await;
3629                self.state.quota.extend(losses);
3630                let (updated, res, _) = results.pop().expect("one job in, one result out");
3631                seat = updated;
3632                let agent_id = seat.agent.clone();
3633                self.state.seats.insert(seat.key.clone(), seat);
3634                let body = match res {
3635                    Ok((body, _)) => body,
3636                    // Skip the seat; a CLI's raw error JSON is never read as
3637                    // this judge's position.
3638                    Err(e) => {
3639                        self.state
3640                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3641                        continue;
3642                    }
3643                };
3644                let tentative = verdict::extract_json::<Position>(&body)
3645                    .ok()
3646                    .and_then(|p| p.tentative)
3647                    .and_then(|s| s.trim().chars().next())
3648                    .map(|c| c.to_ascii_uppercase());
3649                self.state.event(
3650                    "deliberate",
3651                    format!(
3652                        "round {round}: judge {} now favours {}",
3653                        j + 1,
3654                        tentative.map_or("—".to_owned(), |c| c.to_string())
3655                    ),
3656                );
3657                turns.push(DeliberationTurn {
3658                    judge: j + 1,
3659                    agent: agent_id,
3660                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3661                    tentative,
3662                });
3663            }
3664            self.state
3665                .deliberation
3666                .push(DeliberationRound { round, turns });
3667            self.state.save()?;
3668        }
3669
3670        self.state.status = RunStatus::Voting;
3671        self.state.save()?;
3672        Ok(())
3673    }
3674
3675    // ---------------------------------------------------------------- vote
3676
3677    async fn vote(&mut self) -> Result<()> {
3678        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3679        // agent files with `magi task add` name the run that paid for it. The
3680        // prompt overlay is cloned alongside it because the waves borrow it
3681        // while `self` is mutably borrowed by the node's own bookkeeping.
3682        let run_id = self.state.id.clone();
3683        let prompts = self.state.config.prompts.clone();
3684        if !self.state.votes.is_empty() {
3685            return Ok(());
3686        }
3687        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3688        if viable.len() == 1 {
3689            return Ok(());
3690        }
3691        self.state.status = RunStatus::Voting;
3692
3693        let language = self.state.config.graph.language.clone();
3694        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3695        let sessions = self.state.config.graph.sessions;
3696        let artifacts = agent::artifacts_dir(&self.state.dir());
3697        let root = self.state.worktree_root();
3698        let base_short = short(&self.state.base_commit);
3699        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3700
3701        let mut jobs = Vec::new();
3702        let mut seats_at = Vec::new();
3703        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3704            if self
3705                .state
3706                .judgements
3707                .get(j)
3708                .is_some_and(|r| r.failed.is_some())
3709            {
3710                continue;
3711            }
3712            let seat_key = format!("judge-{}", j + 1);
3713            let spec = self.occupant(&seat_key, spec);
3714            let seat = self.seat(&seat_key, &spec.id);
3715            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3716            let text = if has_context(&spec, &seat, sessions) {
3717                prompt::final_vote(&viable, &language)
3718            } else {
3719                full.clone()
3720            };
3721            jobs.push(SeatJob {
3722                spec,
3723                seat,
3724                prompt: text,
3725                cwd: root.join(format!("judge-{}", j + 1)),
3726                timeout,
3727                allow_write: false,
3728                sessions,
3729                artifacts: artifacts.clone(),
3730                stem: format!("vote-judge-{}", j + 1),
3731                handover: Some(full),
3732            });
3733            seats_at.push(j);
3734        }
3735
3736        self.state.event(
3737            "vote",
3738            format!(
3739                "collecting {} final votes one by one, privately",
3740                jobs.len()
3741            ),
3742        );
3743        let allowed = viable.clone();
3744        let mut quota_losses = Vec::new();
3745        let cache = self.state.config.cache_dir();
3746        let ctx = WaveCtx {
3747            carry_seats: false,
3748            pauses: Some((&self.pause, &self.interrupt)),
3749            run: &run_id,
3750            node: "vote",
3751            prompts: &prompts,
3752            cache: cache.as_deref(),
3753            round: None,
3754        };
3755        let results = ask_json_wave::<FinalVote>(
3756            jobs,
3757            Arc::clone(&self.sem),
3758            self.state.config.graph.retries,
3759            &self.roles.judge_roster,
3760            &ctx,
3761            &mut quota_losses,
3762            &mut self.state,
3763            &move |v: &FinalVote| match v.label() {
3764                Some(c) if allowed.contains(&c) => Ok(()),
3765                other => bail!("vote {other:?} is not one of {allowed:?}"),
3766            },
3767        )
3768        .await;
3769        if self.state.resource_block.is_some() {
3770            return self.stop_for_resource();
3771        }
3772        self.state.quota.extend(quota_losses);
3773
3774        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3775            let agent_id = seat.agent.clone();
3776            self.state.seats.insert(seat.key.clone(), seat);
3777            let initial = self
3778                .state
3779                .judgements
3780                .get(j)
3781                .and_then(|r| r.ranking.first().copied());
3782            let mut record = VoteRecord {
3783                judge: j + 1,
3784                agent: agent_id,
3785                vote: None,
3786                reason: String::new(),
3787                changed: false,
3788            };
3789            match res {
3790                Ok((v, _)) => {
3791                    record.vote = v.label();
3792                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3793                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3794                    self.state.event(
3795                        "vote",
3796                        format!(
3797                            "judge {} voted {}{}",
3798                            j + 1,
3799                            record.vote.unwrap_or('?'),
3800                            if record.changed { " (changed)" } else { "" }
3801                        ),
3802                    );
3803                }
3804                Err(e) => {
3805                    self.state
3806                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3807                }
3808            }
3809            self.state.votes.push(record);
3810            self.state.save()?;
3811        }
3812        Ok(())
3813    }
3814
3815    // --------------------------------------------------------------- tally
3816
3817    fn tally(&mut self) -> Result<()> {
3818        if self.state.tally.is_some() {
3819            return Ok(());
3820        }
3821        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3822        let tops: Vec<char> = self
3823            .state
3824            .judgements
3825            .iter()
3826            .filter_map(|j| j.ranking.first().copied())
3827            .collect();
3828        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3829
3830        // A judge whose private vote failed still counted once, in the initial
3831        // ranking; using it beats discarding a whole seat.
3832        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3833        let mut cast: Vec<char> = Vec::new();
3834        for (i, j) in self.state.judgements.iter().enumerate() {
3835            let vote = self
3836                .state
3837                .votes
3838                .iter()
3839                .find(|v| v.judge == i + 1)
3840                .and_then(|v| v.vote)
3841                .or_else(|| j.ranking.first().copied());
3842            if let Some(v) = vote {
3843                *first_choice.entry(v).or_insert(0) += 1;
3844                cast.push(v);
3845            }
3846        }
3847
3848        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3849        for j in &self.state.judgements {
3850            let n = j.ranking.len();
3851            for (pos, label) in j.ranking.iter().enumerate() {
3852                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3853            }
3854        }
3855
3856        let best = first_choice.values().copied().max().unwrap_or(0);
3857        let mut leaders: Vec<char> = first_choice
3858            .iter()
3859            .filter(|(_, v)| **v == best)
3860            .map(|(k, _)| *k)
3861            .collect();
3862        let mut tie_break = None;
3863        if leaders.len() > 1 {
3864            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3865            let borda_leaders: Vec<char> = leaders
3866                .iter()
3867                .copied()
3868                .filter(|l| borda[l] == top_borda)
3869                .collect();
3870            tie_break = Some(if borda_leaders.len() == 1 {
3871                format!(
3872                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3873                    leaders.len()
3874                )
3875            } else {
3876                format!(
3877                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3878                    leaders.len()
3879                )
3880            });
3881            leaders = borda_leaders;
3882            leaders.sort_unstable();
3883        }
3884        let winner = *leaders
3885            .first()
3886            .or(viable.first())
3887            .context("no candidate to declare a winner from")?;
3888
3889        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3890        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3891        let deliberated = !self.state.deliberation.is_empty();
3892
3893        // Whose verdict is this? A rate-limited seat is absent even if it
3894        // ranked before the limit hit, so presence is measured against the
3895        // recorded losses, not just "did a ranking ever appear".
3896        let quota_seats: std::collections::BTreeSet<&str> =
3897            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3898        let mut present = 0usize;
3899        for (i, j) in self.state.judgements.iter().enumerate() {
3900            if quota_seats.contains(j.seat.as_str()) {
3901                continue;
3902            }
3903            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3904            let voted = self
3905                .state
3906                .votes
3907                .iter()
3908                .any(|v| v.judge == i + 1 && v.vote.is_some());
3909            if ranked || voted {
3910                present += 1;
3911            }
3912        }
3913        // Strict majority of the configured panel. A bare majority is real
3914        // signal we can act on, while a minority verdict must never stand in
3915        // for a healthy one. A one-candidate run needs no panel at all, and
3916        // `judges` stays `0` rather than the roster size a panel that never
3917        // sat would otherwise be credited with.
3918        let needs_quorum = viable.len() > 1;
3919        let judges_total = if needs_quorum {
3920            self.roles.judges.len()
3921        } else {
3922            0
3923        };
3924        let quorum = if needs_quorum {
3925            judges_total / 2 + 1
3926        } else {
3927            0
3928        };
3929        let met_quorum = !needs_quorum || present >= quorum;
3930        let uncontested = (!needs_quorum).then(|| {
3931            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3932        });
3933
3934        self.state.event(
3935            "tally",
3936            match &uncontested {
3937                Some(reason) => format!("winner {winner} — {reason}"),
3938                None => format!(
3939                    "winner {winner} — votes {} | initial {} | {} changed | \
3940                     {present}/{judges_total} judges{}",
3941                    first_choice
3942                        .iter()
3943                        .map(|(k, v)| format!("{k}:{v}"))
3944                        .collect::<Vec<_>>()
3945                        .join(" "),
3946                    if unanimous_initial {
3947                        "unanimous"
3948                    } else {
3949                        "split"
3950                    },
3951                    changed_votes,
3952                    if met_quorum {
3953                        String::new()
3954                    } else {
3955                        format!(" — below quorum ({quorum} required)")
3956                    },
3957                ),
3958            },
3959        );
3960        if !met_quorum {
3961            self.state.event(
3962                "stall",
3963                format!(
3964                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3965                     the run stops here, resumable"
3966                ),
3967            );
3968        }
3969        self.state.tally = Some(Tally {
3970            first_choice,
3971            borda,
3972            winner,
3973            rankings: tops.len(),
3974            unanimous_initial,
3975            deliberated,
3976            changed_votes,
3977            unanimous_final,
3978            tie_break,
3979            judges: judges_total,
3980            present,
3981            quorum,
3982            met_quorum,
3983            uncontested,
3984        });
3985        self.state.status = if met_quorum {
3986            RunStatus::Reviewing
3987        } else {
3988            RunStatus::Stalled
3989        };
3990        self.state.save()?;
3991        Ok(())
3992    }
3993
3994    // ------------------------------------------------------------- recover
3995
3996    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3997    /// resumed toward completion once the transient cause clears.
3998    ///
3999    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4000    /// it toward the quorum, which is exactly the set of seats whose absence
4001    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4002    /// not depend on which node happened to hit the limit), or an ordinary
4003    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4004    /// seat is never disturbed.
4005    ///
4006    /// A seat that now answers with a usable ranking is "recovered": its
4007    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4008    /// `tally` counts it present again), and its vote re-collected. A seat that
4009    /// still fails keeps its loss and stays absent.
4010    ///
4011    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4012    /// to review/gate/merge), `false` when it is still below quorum (the run
4013    /// stays `Stalled`, still resumable for a later retry).
4014    #[allow(clippy::too_many_lines)]
4015    async fn recover_stall(&mut self) -> Result<bool> {
4016        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4017        // agent files with `magi task add` name the run that paid for it. The
4018        // prompt overlay is cloned alongside it because the waves borrow it
4019        // while `self` is mutably borrowed by the node's own bookkeeping.
4020        let run_id = self.state.id.clone();
4021        let prompts = self.state.config.prompts.clone();
4022        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4023        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4024        // is what keeps a non-quota collapse — or a quota loss recorded at the
4025        // deliberate node — from being a permanent dead-end on `--resume`.
4026        let quota_seats: BTreeSet<&str> =
4027            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4028        let absent: Vec<String> = self
4029            .state
4030            .judgements
4031            .iter()
4032            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4033            .map(|j| j.seat.clone())
4034            .collect();
4035        if absent.is_empty() {
4036            return Ok(false);
4037        }
4038        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4039        if viable.len() <= 1 {
4040            return Ok(false);
4041        }
4042        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4043        let language = self.state.config.graph.language.clone();
4044        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4045        let sessions = self.state.config.graph.sessions;
4046        let artifacts = agent::artifacts_dir(&self.state.dir());
4047        let root = self.state.worktree_root();
4048        let base_short = short(&self.state.base_commit);
4049        let candidates: Vec<Candidate> = viable.clone();
4050
4051        // Map each absent seat key to its 0-based position in `roles.judges`.
4052        let mut positions: Vec<usize> = absent
4053            .iter()
4054            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4055            .collect();
4056        if positions.is_empty() {
4057            return Ok(false);
4058        }
4059        positions.sort_unstable();
4060        positions.dedup();
4061
4062        // Re-rank the lost seats, one blind prompt each.
4063        let mut judge_jobs = Vec::new();
4064        for &j in &positions {
4065            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4066            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4067            let seat_key = format!("judge-{}", j + 1);
4068            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4069            let seat = self.seat(&seat_key, &spec.id);
4070            judge_jobs.push(SeatJob {
4071                spec,
4072                seat,
4073                prompt: prompt::judge(
4074                    &self.state.instruction,
4075                    &views,
4076                    self.roles.judges.len(),
4077                    &base_short,
4078                    &language,
4079                ),
4080                cwd: root.join(seat_key),
4081                timeout,
4082                allow_write: false,
4083                sessions,
4084                artifacts: artifacts.clone(),
4085                stem: format!("judge-{}-recover", j + 1),
4086                handover: None,
4087            });
4088        }
4089
4090        let labels_for_check = labels.clone();
4091        let mut judge_losses = Vec::new();
4092        let retries = self.state.config.graph.retries;
4093        let cache = self.state.config.cache_dir();
4094        let ctx = WaveCtx {
4095            carry_seats: false,
4096            pauses: Some((&self.pause, &self.interrupt)),
4097            run: &run_id,
4098            node: "judge",
4099            prompts: &prompts,
4100            cache: cache.as_deref(),
4101            round: None,
4102        };
4103        let results = ask_json_wave::<Ranking>(
4104            judge_jobs,
4105            Arc::clone(&self.sem),
4106            retries,
4107            &self.roles.judge_roster,
4108            &ctx,
4109            &mut judge_losses,
4110            &mut self.state,
4111            &move |r: &Ranking| r.validate(&labels_for_check),
4112        )
4113        .await;
4114        if self.state.resource_block.is_some() {
4115            self.stop_for_resource()?;
4116            return Ok(false);
4117        }
4118
4119        // Refresh the judgement of every seat that ranked again.
4120        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4121        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4122            let agent_id = seat.agent.clone();
4123            self.state.seats.insert(seat.key.clone(), seat);
4124            let record = &mut self.state.judgements[j];
4125            match res {
4126                Ok((ranking, out)) => {
4127                    record.agent = agent_id;
4128                    record.ranking = ranking.normalized();
4129                    record.reasons = ranking.reasons;
4130                    record.strengths = ranking.strengths;
4131                    record.confidence = ranking.confidence;
4132                    record.failed = None;
4133                    record.duration_ms = out.duration_ms;
4134                    recovered.insert(j);
4135                    self.state.event(
4136                        "recover",
4137                        format!("judge {} ranked again after the limit", j + 1),
4138                    );
4139                }
4140                Err(e) => {
4141                    self.state
4142                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4143                }
4144            }
4145        }
4146
4147        // Re-ask the votes of the seats that recovered a ranking.
4148        let mut vote_jobs = Vec::new();
4149        let mut vote_pos: Vec<usize> = Vec::new();
4150        for &j in &recovered {
4151            let seat_key = format!("judge-{}", j + 1);
4152            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4153            let seat = self.seat(&seat_key, &spec.id);
4154            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4155            let text = if has_context(&spec, &seat, sessions) {
4156                prompt::final_vote(&labels, &language)
4157            } else {
4158                full.clone()
4159            };
4160            vote_jobs.push(SeatJob {
4161                spec,
4162                seat,
4163                prompt: text,
4164                cwd: root.join(seat_key),
4165                timeout,
4166                allow_write: false,
4167                sessions,
4168                artifacts: artifacts.clone(),
4169                stem: format!("vote-judge-{}-recover", j + 1),
4170                handover: Some(full),
4171            });
4172            vote_pos.push(j);
4173        }
4174        let allowed = labels.clone();
4175        let mut vote_losses = Vec::new();
4176        let vote_retries = self.state.config.graph.retries;
4177        let vote_cache = self.state.config.cache_dir();
4178        let ctx = WaveCtx {
4179            carry_seats: false,
4180            pauses: Some((&self.pause, &self.interrupt)),
4181            run: &run_id,
4182            node: "vote",
4183            prompts: &prompts,
4184            cache: vote_cache.as_deref(),
4185            round: None,
4186        };
4187        let votes = ask_json_wave::<FinalVote>(
4188            vote_jobs,
4189            Arc::clone(&self.sem),
4190            vote_retries,
4191            &self.roles.judge_roster,
4192            &ctx,
4193            &mut vote_losses,
4194            &mut self.state,
4195            &move |v: &FinalVote| match v.label() {
4196                Some(c) if allowed.contains(&c) => Ok(()),
4197                other => bail!("vote {other:?} is not one of {allowed:?}"),
4198            },
4199        )
4200        .await;
4201        if self.state.resource_block.is_some() {
4202            self.stop_for_resource()?;
4203            return Ok(false);
4204        }
4205        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4206            let agent_id = seat.agent.clone();
4207            self.state.seats.insert(seat.key.clone(), seat);
4208            match res {
4209                Ok((v, _)) => {
4210                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4211                        rec.vote = v.label();
4212                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4213                    } else {
4214                        self.state.votes.push(VoteRecord {
4215                            judge: j + 1,
4216                            agent: agent_id,
4217                            vote: v.label(),
4218                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4219                            changed: false,
4220                        });
4221                    }
4222                    self.state.event(
4223                        "recover",
4224                        format!("judge {} voted again after the limit", j + 1),
4225                    );
4226                }
4227                Err(e) => {
4228                    self.state
4229                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4230                }
4231            }
4232        }
4233
4234        // A seat that ranked again is present even if its re-vote failed —
4235        // `tally` falls back to the initial ranking's first choice — so clear
4236        // its quota loss. Seats that still fail keep theirs and stay absent.
4237        let recovered_keys: BTreeSet<String> = recovered
4238            .iter()
4239            .map(|&j| format!("judge-{}", j + 1))
4240            .collect();
4241        self.state
4242            .quota
4243            .retain(|q| !recovered_keys.contains(&q.seat));
4244        // A seat that hit the limit again is a fresh loss, not the old one:
4245        // replace the stale entry so the history stays one-per-seat and the
4246        // daemon can tell this attempt's loss from a previous session's.
4247        for loss in judge_losses.into_iter().chain(vote_losses) {
4248            if recovered_keys.contains(&loss.seat) {
4249                continue;
4250            }
4251            self.state.quota.retain(|q| q.seat != loss.seat);
4252            self.state.quota.push(loss);
4253        }
4254
4255        // Recompute the verdict from the refreshed panel.
4256        self.state.tally = None;
4257        self.tally()?;
4258        Ok(self
4259            .state
4260            .tally
4261            .as_ref()
4262            .map(|t| t.met_quorum)
4263            .unwrap_or(false))
4264    }
4265
4266    // ---------------------------------------------------------------- graft
4267
4268    /// Port the strengths the judges attributed to the losing candidates onto
4269    /// the winner, once, between the tally and the review loop.
4270    ///
4271    /// Best-effort by construction: it returns nothing and fails nothing. A
4272    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4273    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4274    /// the run carries on as if the phase were off. See [`graft`] for the
4275    /// decisions and `AGENTS.md` for the invariants.
4276    ///
4277    /// The one exception is a rollback that could not be confirmed: the
4278    /// rejected commit may still be the winner's tip, so the run is blocked
4279    /// (resumable; the record stays `Running`, which makes a resume retry the
4280    /// restore) instead of reviewing, gating and merging it.
4281    async fn graft(&mut self) -> Result<()> {
4282        let Err(e) = self.graft_inner().await else {
4283            return Ok(());
4284        };
4285        if e.is::<crate::repo_lock::ResourceBlocked>() {
4286            return Err(e);
4287        }
4288        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4289            self.state.status = RunStatus::Blocked;
4290            self.state.event("graft", format!("blocked: {e:#}"));
4291            self.state.save()?;
4292            return Err(e);
4293        }
4294        // An error after the `Running` record was saved may have left the
4295        // fixer's work in the tree: the record must not stay unsettled
4296        // (a later resume would reset past the review fixes), so confirm the
4297        // restore now, or block.
4298        if let Some(rec) = self.state.graft.clone()
4299            && !rec.outcome.settled()
4300        {
4301            if let Some(winner) = self.state.winner().cloned() {
4302                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4303                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4304                        return Err(re);
4305                    }
4306                    self.state.status = RunStatus::Blocked;
4307                    self.state.event("graft", format!("blocked: {re:#}"));
4308                    self.state.save()?;
4309                    return Err(re);
4310                }
4311            }
4312            if let Some(g) = self.state.graft.as_mut() {
4313                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4314            }
4315        }
4316        self.state.event("graft", format!("skipped: {e:#}"));
4317        if let Err(e) = self.state.save() {
4318            tracing::warn!("save after graft: {e:#}");
4319        }
4320        Ok(())
4321    }
4322
4323    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4324    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4325    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4326        let _coordination = self.coordinate(wt).await?;
4327        crate::recovery::restore_repair(wt, pre)
4328            .await
4329            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4330    }
4331
4332    /// A skip is a run event, recorded once however often the node is re-entered.
4333    fn graft_skipped(&mut self, why: &str) {
4334        let message = format!("skipped: {why}");
4335        if !self
4336            .state
4337            .events
4338            .iter()
4339            .any(|e| e.node == "graft" && e.message == message)
4340        {
4341            self.state.event("graft", message);
4342        }
4343    }
4344
4345    async fn graft_inner(&mut self) -> Result<()> {
4346        // A round that was started and never finished (crash, kill): the
4347        // fixer may have left anything in the tree, so put the winner back.
4348        if let Some(rec) = self.state.graft.clone()
4349            && !rec.outcome.settled()
4350        {
4351            // Reviews exist only if the graft ended without its record being
4352            // settled (a swallowed error): the tree has moved on since, and
4353            // a reset to `pre` would discard the review fixes.
4354            if self.state.reviews.is_empty()
4355                && let Some(winner) = self.state.winner().cloned()
4356            {
4357                self.graft_restore(&winner.worktree, &rec.pre).await?;
4358            }
4359            if rec.outcome == GraftOutcome::WaitingForRestore {
4360                self.state.graft = None;
4361                self.state.save()?;
4362            } else {
4363                if let Some(g) = self.state.graft.as_mut() {
4364                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4365                }
4366                self.state.event(
4367                    "graft",
4368                    "rolled back: the previous round was interrupted; the winner is untouched",
4369                );
4370                self.state.save()?;
4371                return Ok(());
4372            }
4373        }
4374        let plan = match graft::plan(&self.state) {
4375            Ok(plan) => plan,
4376            Err(skip) => {
4377                if skip != graft::Skip::AlreadyDone {
4378                    self.graft_skipped(skip.reason());
4379                    self.state.save()?;
4380                }
4381                return Ok(());
4382            }
4383        };
4384        let Some(winner) = self.state.winner().cloned() else {
4385            return Ok(());
4386        };
4387        let wt = winner.worktree.clone();
4388        if !git::is_clean(&wt).await? {
4389            self.graft_skipped("the winner's worktree has uncommitted changes");
4390            self.state.save()?;
4391            return Ok(());
4392        }
4393        let pre = git::rev_parse(&wt, "HEAD").await?;
4394        // Saved before the fixer is called: a stop from here on is repaired
4395        // by the branch above on the next entry.
4396        self.state.graft = Some(GraftRecord {
4397            pre: pre.clone(),
4398            post: None,
4399            lines: None,
4400            limit: None,
4401            agent: None,
4402            outcome: GraftOutcome::Running,
4403            requested: graft::requested(&plan),
4404            adoption: None,
4405            adoption_error: None,
4406        });
4407        self.state.event(
4408            "graft",
4409            format!(
4410                "porting strengths of candidate(s) {} onto {}",
4411                plan.strengths
4412                    .iter()
4413                    .map(|(l, _, _)| l.to_string())
4414                    .collect::<Vec<_>>()
4415                    .join(", "),
4416                plan.winner
4417            ),
4418        );
4419        self.state.save()?;
4420
4421        let fix_prompt = prompt::graft(
4422            &self.state.instruction,
4423            plan.winner,
4424            &plan.strengths,
4425            &self.state.config.graph.language,
4426        );
4427        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4428        let sessions = self.state.config.graph.sessions;
4429        let artifacts = agent::artifacts_dir(&self.state.dir());
4430        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4431        let (_, seat, out) = self
4432            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4433                SeatJob {
4434                    prompt: fix_prompt.clone(),
4435                    spec,
4436                    seat,
4437                    cwd: wt.clone(),
4438                    timeout,
4439                    allow_write: true,
4440                    sessions,
4441                    artifacts: artifacts.clone(),
4442                    stem: "graft".to_owned(),
4443                    handover: None,
4444                }
4445            })
4446            .await;
4447        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4448            // Preserve the need to restore earlier fallback edits without
4449            // confusing an unstarted seat with an interrupted invocation.
4450            self.state.graft.as_mut().expect("graft record").outcome =
4451                GraftOutcome::WaitingForRestore;
4452            return self.stop_for_resource();
4453        }
4454        let mut why: Option<String> = None;
4455        match &out {
4456            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4457                why = Some("the fixer produced nothing usable".to_owned());
4458            }
4459            AgentOutcome::Ok(_) => {}
4460            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4461            AgentOutcome::Quota(o) => {
4462                self.state.quota.push(QuotaLoss {
4463                    seat: seat.key.clone(),
4464                    node: "graft".to_owned(),
4465                    at: Timestamp::now(),
4466                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4467                });
4468                why = Some("rate limited (quota)".to_owned());
4469            }
4470            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4471                why = Some(format!(
4472                    "the fixer failed: {}",
4473                    e.lines()
4474                        .next()
4475                        .unwrap_or("")
4476                        .chars()
4477                        .take(160)
4478                        .collect::<String>()
4479                ));
4480            }
4481        }
4482        let agent_id = seat.agent.clone();
4483        self.state.seats.insert(seat.key.clone(), seat);
4484
4485        let mut post = None;
4486        let mut lines = None;
4487        let _coordination = self.coordinate(&wt).await?;
4488        if why.is_none() {
4489            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4490            if let Ok(r) = &rescued {
4491                self.state.note_withheld("graft", &r.withheld);
4492            }
4493            let head = git::rev_parse(&wt, "HEAD").await?;
4494            // What is measured must be what gets reviewed and gated: a failed
4495            // rescue, or anything still uncommitted (withheld lockfiles
4496            // aside, which are never part of the tree), rejects the graft.
4497            let dirty = match &rescued {
4498                Ok(r) => git::status_porcelain(&wt)
4499                    .await
4500                    .map(|st| {
4501                        st.lines().any(|l| {
4502                            !r.withheld
4503                                .iter()
4504                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4505                        })
4506                    })
4507                    .unwrap_or(true),
4508                Err(_) => true,
4509            };
4510            if dirty {
4511                why = Some("the fixer left changes that could not be committed".to_owned());
4512            } else if head == pre {
4513                why = Some("the fixer changed nothing".to_owned());
4514            } else {
4515                lines = git::changed_lines(&wt, &pre, &head).await;
4516                post = Some(head);
4517            }
4518        }
4519        let (adoption, adoption_error) = match &out {
4520            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4521            _ => (None, None),
4522        };
4523        let requested = graft::requested(&plan);
4524        let outcome = match why {
4525            None => {
4526                self.state.event(
4527                    "graft",
4528                    format!(
4529                        "applied: {}",
4530                        lines.map_or_else(
4531                            || "changed lines unavailable".to_owned(),
4532                            |n| format!("{n} changed lines")
4533                        )
4534                    ),
4535                );
4536                GraftOutcome::Applied
4537            }
4538            Some(why) => {
4539                // Not settled until the restore is confirmed: the record
4540                // stays `Running`, so a resume retries it.
4541                crate::recovery::restore_repair(&wt, &pre)
4542                    .await
4543                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4544                self.state.event(
4545                    "graft",
4546                    format!("rolled back: {why}; the winner is untouched"),
4547                );
4548                GraftOutcome::RolledBack(why)
4549            }
4550        };
4551        self.state.graft = Some(GraftRecord {
4552            pre,
4553            post,
4554            lines,
4555            limit: None,
4556            agent: Some(agent_id),
4557            outcome,
4558            requested,
4559            adoption,
4560            adoption_error,
4561        });
4562        self.state.save()?;
4563        Ok(())
4564    }
4565
4566    // ----------------------------------------------------------------- fold
4567
4568    async fn fold_losers(&mut self) -> Result<()> {
4569        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4570            return Ok(());
4571        };
4572        if !self.state.candidate_preservation.is_empty() {
4573            bail!("candidate preservation incomplete; refusing cleanup");
4574        }
4575        let repo = self.state.repo.clone();
4576        let _coordination = self.coordinate(&repo).await?;
4577        let mut folded = Vec::new();
4578        for i in 0..self.state.candidates.len() {
4579            let c = &self.state.candidates[i];
4580            if c.label == winner || c.folded {
4581                continue;
4582            }
4583            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4584            remove_preserved_worktree(&repo, &wt).await?;
4585            if git::branch_exists(&repo, &branch).await?
4586                && !git::branch_delete(&repo, &branch).await?
4587            {
4588                bail!("candidate branch removal refused");
4589            }
4590            self.state.candidates[i].folded = true;
4591            self.state.save()?;
4592            folded.push(label.to_string());
4593        }
4594        // The judges are finished; their checkouts are pure cost from here.
4595        let root = self.state.worktree_root();
4596        for j in 1..=self.roles.judges.len() {
4597            let wt = root.join(format!("judge-{j}"));
4598            if wt.exists() {
4599                remove_preserved_worktree(&repo, &wt).await?;
4600            }
4601        }
4602        // The design-deliberation stage is finished by the time a tally
4603        // exists — same reasoning as the judges above.
4604        if self.state.config.graph.advise {
4605            for k in 1..=self.state.config.graph.advisors {
4606                let wt = root.join(format!("advisor-{k}"));
4607                if wt.exists() {
4608                    remove_preserved_worktree(&repo, &wt).await?;
4609                }
4610            }
4611        }
4612        if !folded.is_empty() {
4613            self.state
4614                .event("fold", format!("folded candidates {}", folded.join(", ")));
4615            self.state.save()?;
4616        }
4617        Ok(())
4618    }
4619
4620    // ------------------------------------------------------------ base sync
4621
4622    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4623    /// anything verifies it.
4624    ///
4625    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4626    /// read whatever is checked out in the winner's worktree. Left alone that
4627    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4628    /// it when the run *branched* - and a run takes long enough that the base
4629    /// has usually moved by the time it gets here. A gate that ran there
4630    /// answers "green on the commit this run started from", not "green on
4631    /// what is about to land", and the difference showed up three times in
4632    /// one day as a green run whose merge would have reverted a file another
4633    /// pull request had already landed.
4634    ///
4635    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4636    /// `land::Step::Rebase` calls, rather than a second implementation of the
4637    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4638    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4639    /// doc warns about ("two answers to a question nobody notices until a
4640    /// diff is wrong").
4641    ///
4642    /// A conflict is not the end of the road: the standing rebase is handed
4643    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4644    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4645    /// land). Once it finishes, review and the gate run as usual on the
4646    /// rebased tree, which is where a breakage the new base caused is caught
4647    /// by the ordinary gate-fix round. magi resolves nothing itself.
4648    ///
4649    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4650    /// `state.base_sync.attempts`, is how many times the base is *rebased
4651    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4652    /// *conflict* was given to a fixer. When the fixer cannot finish the
4653    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4654    /// what was tried (rounds spent, paths still conflicted) and the branch
4655    /// and worktree stay exactly as they were - untouched, for a person to
4656    /// look at - which is also what makes re-entering this function
4657    /// afterwards a no-op instead of a second attempt at the same wall. A
4658    /// push failure ends the same way.
4659    async fn sync_to_base(&mut self) -> Result<()> {
4660        if self.state.status == RunStatus::AlreadyInBase {
4661            return Ok(());
4662        }
4663        let conflicted = self
4664            .state
4665            .base_sync
4666            .as_ref()
4667            .is_some_and(|s| s.conflict.is_some());
4668        let Some(winner) = self.state.winner().cloned() else {
4669            return Ok(());
4670        };
4671
4672        let repo = self.state.repo.clone();
4673        let remote = self.state.config.merge.remote.clone();
4674        let base_branch = self.state.base_branch.clone();
4675        let tracking = format!("{remote}/{base_branch}");
4676
4677        git::fetch(&repo, &remote, &base_branch).await.ok();
4678        // No network, or the remote never had this branch: the run already
4679        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4680        // otherwise), and one that got this far is not blocked by a fetch
4681        // that fails now. It just has no newer tip to compare against.
4682        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4683            return Ok(());
4684        };
4685
4686        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4687        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4688        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4689
4690        // Before any rebase, and before a recorded conflict is honoured: a
4691        // branch whose change reached the base under other commit ids has
4692        // nothing to rebase and nothing to conflict with, and a run that
4693        // already stopped on that phantom conflict recovers here on resume.
4694        // `behind == 0` with head == tip is a branch the base has since taken
4695        // in whole, whether or not a conflict was ever recorded: the ancestry
4696        // proof must still run (`classify` ignores a head still on the start
4697        // commit).
4698        if (behind > 0 || conflicted || head == tip)
4699            && self
4700                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4701                .await?
4702        {
4703            return Ok(());
4704        }
4705        if conflicted {
4706            return Ok(());
4707        }
4708
4709        if behind == 0 {
4710            // A fixer-finished rebase moves the branch ref before the
4711            // winner's worktree is told (`sync_to_head` below). A run that
4712            // died in between resumes here with `behind == 0` and a tree still
4713            // holding the pre-rebase files, which review and the gate would
4714            // then read. That state is exactly: HEAD moved off the tip the
4715            // rebase started from, yet the tree is still identical to that
4716            // tip. A tree with edits of its own differs from it, so nothing
4717            // is thrown away.
4718            if let Some(from) = self
4719                .state
4720                .rebase_fixes
4721                .iter()
4722                .rev()
4723                .find_map(|r| r.from.clone())
4724                && from != head
4725                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4726                    .await
4727                    .is_ok_and(|o| o.ok())
4728            {
4729                git::sync_to_head(&winner.worktree).await?;
4730            }
4731            // An earlier attempt may have rebased the branch locally and died
4732            // before pushing it (only the fresh-rebase arm below pushes).
4733            // Publish it now, so the plain push at PR time is not refused as
4734            // a non-fast-forward. A run already holding a recorded conflict
4735            // never reaches here; that case is out of scope.
4736            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4737            if let Some(why) = &conflict {
4738                self.state.status = RunStatus::Blocked;
4739                self.state.event("land", why.clone());
4740            }
4741            self.state.base_sync = Some(BaseSync {
4742                tip,
4743                behind: 0,
4744                attempts,
4745                conflict,
4746                already_in: None,
4747            });
4748            self.state.save()?;
4749            return Ok(());
4750        }
4751
4752        if attempts >= BASE_SYNC_ROUNDS {
4753            let why = format!(
4754                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4755                 rebase(s); rebasing again would only race it",
4756                winner.branch
4757            );
4758            self.state.status = RunStatus::Blocked;
4759            self.state.base_sync = Some(BaseSync {
4760                tip,
4761                behind,
4762                attempts,
4763                conflict: Some(why.clone()),
4764                already_in: None,
4765            });
4766            self.state.event("land", why);
4767            self.state.save()?;
4768            return Ok(());
4769        }
4770
4771        self.state.event(
4772            "land",
4773            format!(
4774                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4775                winner.branch
4776            ),
4777        );
4778        self.state.save()?;
4779
4780        // The remote's copy of the branch, read now and only if the fetch
4781        // really succeeded (a stale tracking ref must never pin a lease). It is
4782        // pushed over after a rebase only when it is a commit this branch
4783        // already contains, by ancestry or by patch (an earlier rebase of ours
4784        // that never reached the remote): anything else is somebody else's work.
4785        let branch_tracking = format!("{remote}/{}", winner.branch);
4786        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4787        let remote_tip = if self.state.task_pr_branch().is_none()
4788            && matches!(&fetched_branch, Ok(o) if o.ok())
4789        {
4790            git::rev_parse(&repo, &branch_tracking).await.ok()
4791        } else {
4792            None
4793        };
4794        // A remote tip this branch does not contain is somebody else's work:
4795        // rebasing would leave a local tip that can never be pushed. Stop
4796        // before touching anything and say so.
4797        if let Some(theirs) = &remote_tip
4798            && !git::is_ancestor(&repo, theirs, &head).await
4799            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4800                .await
4801                .is_ok_and(|missing| missing.is_empty())
4802        {
4803            let why = format!(
4804                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4805                 them",
4806                short(theirs),
4807                winner.branch
4808            );
4809            self.state.status = RunStatus::Blocked;
4810            self.state.base_sync = Some(BaseSync {
4811                tip,
4812                behind,
4813                attempts,
4814                conflict: Some(why.clone()),
4815                already_in: None,
4816            });
4817            self.state.event("land", why);
4818            self.state.save()?;
4819            return Ok(());
4820        }
4821
4822        let scratch = self.state.dir().join("base-sync");
4823        let rebased = match crate::rebase::rebase_with_fixer(
4824            &mut self.state,
4825            &scratch,
4826            &winner.branch,
4827            &tracking,
4828        )
4829        .await
4830        {
4831            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4832            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4833            Err(e) => Err(e),
4834        };
4835        let attempts = attempts + 1;
4836        match rebased {
4837            Ok(None) => {
4838                // The branch ref moved, but a worktree that already had it
4839                // checked out (the winner's) was not told; sync its index and
4840                // files before anything reads them.
4841                git::sync_to_head(&winner.worktree).await?;
4842                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4843                let mut conflict = None;
4844                if let Some(pinned) = &remote_tip {
4845                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4846                    match pushed {
4847                        Ok(o) if o.ok() => self.state.event(
4848                            "land",
4849                            format!("pushed rebased {} to {remote}", winner.branch),
4850                        ),
4851                        Ok(o) => {
4852                            conflict = Some(format!(
4853                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4854                                winner.branch,
4855                                short(pinned),
4856                                o.stderr.chars().take(600).collect::<String>()
4857                            ));
4858                        }
4859                        Err(e) => {
4860                            conflict = Some(format!(
4861                                "rebased {} locally but could not push it: {e:#}",
4862                                winner.branch
4863                            ));
4864                        }
4865                    }
4866                }
4867                if let Some(why) = &conflict {
4868                    self.state.status = RunStatus::Blocked;
4869                    self.state.event("land", why.clone());
4870                }
4871                self.state.base_sync = Some(BaseSync {
4872                    tip: tip.clone(),
4873                    behind: 0,
4874                    attempts,
4875                    conflict,
4876                    already_in: None,
4877                });
4878                self.state
4879                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4880            }
4881            Ok(Some(conflict)) => {
4882                let why = format!(
4883                    "{} conflicts with {tracking} and did not rebase: {}",
4884                    winner.branch,
4885                    conflict.chars().take(600).collect::<String>()
4886                );
4887                self.state.status = RunStatus::Blocked;
4888                self.state.base_sync = Some(BaseSync {
4889                    tip,
4890                    behind,
4891                    attempts,
4892                    conflict: Some(why.clone()),
4893                    already_in: None,
4894                });
4895                self.state.event("land", why);
4896            }
4897            // A refused Git write or a busy repository: nothing is wrong with
4898            // the branch. The scratch rebase stays standing and the run
4899            // resumes into it (`base_sync` must not claim a conflict).
4900            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4901                self.state.save()?;
4902                return Ok(());
4903            }
4904            Err(e) => {
4905                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4906                self.state.status = RunStatus::Blocked;
4907                self.state.base_sync = Some(BaseSync {
4908                    tip,
4909                    behind,
4910                    attempts,
4911                    conflict: Some(why.clone()),
4912                    already_in: None,
4913                });
4914                self.state.event("land", why);
4915            }
4916        }
4917        self.state.save()?;
4918        Ok(())
4919    }
4920
4921    /// Push a branch an earlier attempt rebased locally but never published,
4922    /// pinned to the remote tip read right after a successful fetch. Returns
4923    /// the reason when the run must stop; `None` when there was nothing to do
4924    /// (no remote copy, the same tip, or a remote copy this branch already
4925    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4926    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4927        // Task competitions publish only after the PR text gate, to their
4928        // public head; never resurrect a candidate branch on the remote.
4929        if self.state.task_pr_branch().is_some() {
4930            return None;
4931        }
4932        let repo = self.state.repo.clone();
4933        let remote = self.state.config.merge.remote.clone();
4934        let fetched = git::fetch(&repo, &remote, branch).await;
4935        if !matches!(&fetched, Ok(o) if o.ok()) {
4936            return None;
4937        }
4938        let branch_tracking = format!("{remote}/{branch}");
4939        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4940        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4941            return None;
4942        }
4943        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4944            .await
4945            .is_ok_and(|missing| missing.is_empty())
4946        {
4947            return Some(format!(
4948                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4949                 them",
4950                short(&theirs)
4951            ));
4952        }
4953        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4954            Ok(o) if o.ok() => {
4955                self.state
4956                    .event("land", format!("pushed rebased {branch} to {remote}"));
4957                None
4958            }
4959            Ok(o) => Some(format!(
4960                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4961                 someone may have pushed): {}",
4962                short(&theirs),
4963                o.stderr.chars().take(600).collect::<String>()
4964            )),
4965            Err(e) => Some(format!(
4966                "{branch} is rebased locally but could not be pushed: {e:#}"
4967            )),
4968        }
4969    }
4970
4971    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4972    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4973    /// whether it did.
4974    ///
4975    /// Checked only when the base is ahead of the branch. A failing check is
4976    /// "not proven" - the ordinary rebase path then decides - never a reason to
4977    /// stop the run.
4978    ///
4979    /// The remote copy of the branch is held to the same standard as the local
4980    /// one: if it carries a tip this worktree does not, that tip must itself be
4981    /// proven in the base, or nothing is settled (a pull request would
4982    /// otherwise be closed over commits nobody checked). The pull request is
4983    /// closed *before* the terminal status is saved; if that fails for a
4984    /// reason other than a refusal (no network, a `gh` error) the run is left
4985    /// `Blocked` with the reason as its conflict, which a resume retries -
4986    /// the same recovery a phantom conflict gets.
4987    async fn settle_already_in(
4988        &mut self,
4989        branch: &str,
4990        tip: &str,
4991        head: &str,
4992        attempts: usize,
4993        behind: usize,
4994    ) -> Result<bool> {
4995        let repo = self.state.repo.clone();
4996        let remote = self.state.config.merge.remote.clone();
4997        let start = self.state.base_commit.clone();
4998        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4999            Ok(Some(e)) => e,
5000            Ok(None) => return Ok(false),
5001            Err(e) => {
5002                tracing::warn!("already-in-base check for {branch}: {e:#}");
5003                return Ok(false);
5004            }
5005        };
5006        let mut verified = vec![head.to_owned()];
5007        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5008        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5009        if matches!(&fetched, Ok(o) if o.ok())
5010            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5011            && theirs != head
5012        {
5013            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5014                Ok(Some(_)) => verified.push(theirs),
5015                _ => return Ok(false),
5016            }
5017        }
5018        let base_branch = self.state.base_branch.clone();
5019        let message = format!(
5020            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5021             land",
5022            evidence.names(),
5023            evidence.proof.as_str()
5024        );
5025        let closed =
5026            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5027                .await;
5028        match closed {
5029            Ok(Ok(url)) => self
5030                .state
5031                .event("land", format!("closed {url}: superseded on {base_branch}")),
5032            Ok(Err(why)) => self
5033                .state
5034                .event("land", format!("did not close a pull request: {why}")),
5035            Err(e) => {
5036                let why = format!(
5037                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5038                     not be closed ({e:#}); resume to retry"
5039                );
5040                self.state.status = RunStatus::Blocked;
5041                self.state.base_sync = Some(BaseSync {
5042                    tip: tip.to_owned(),
5043                    behind,
5044                    attempts,
5045                    conflict: Some(why.clone()),
5046                    already_in: None,
5047                });
5048                self.state.event("land", why);
5049                self.state.save()?;
5050                return Ok(true);
5051            }
5052        }
5053        self.state.status = RunStatus::AlreadyInBase;
5054        self.state.base_sync = Some(BaseSync {
5055            tip: tip.to_owned(),
5056            behind,
5057            attempts,
5058            conflict: None,
5059            already_in: Some(evidence),
5060        });
5061        self.state.event("land", message);
5062        self.state.save()?;
5063        self.settle_questions();
5064        Ok(true)
5065    }
5066
5067    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5068    /// last landed the winner on, once it has run, else the commit the run
5069    /// branched from.
5070    ///
5071    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5072    /// and `vote` all happen before there is a winner to rebase, so they
5073    /// compare every candidate against the branch point on purpose, and a
5074    /// base that moves after they are already done cannot change an answer
5075    /// they already gave.
5076    fn landing_base(&self) -> String {
5077        self.state
5078            .base_sync
5079            .as_ref()
5080            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5081    }
5082
5083    // ------------------------------------------------------- operator fix
5084
5085    /// Route specific, already-recorded review findings to a fixer for a
5086    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5087    /// entry point.
5088    ///
5089    /// Distinct from `review_loop`'s own fix step in three ways: it never
5090    /// runs a reviewer wave, it never spends review-round budget, and what
5091    /// happened is recorded as an [`OperatorFixRequest`] appended to
5092    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5093    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5094    /// and vote must never be rewritten to look like a manufactured blocking
5095    /// verdict.
5096    ///
5097    /// Only meaningful once review has actually concluded: `Ready` (handed
5098    /// off with findings still open, or simply concluded clean while minor
5099    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5100    /// gate failed). Everything else is refused: a run still in progress
5101    /// should simply be resumed, and a `Merged` run's branch has already
5102    /// landed — reopening *this* run's own record cannot change that, so the
5103    /// answer there is a fresh `magi review <branch>`.
5104    ///
5105    /// A real commit here re-verifies through a fresh, ordinary review-only
5106    /// run on the same branch ([`Self::review`]) rather than reopening this
5107    /// run's own `review_loop`: once any round in this run's history went
5108    /// clean, `review_conclusion` treats that as permanent by design (the
5109    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5110    /// way to force one more genuine reviewer wave out of *this* run without
5111    /// either rewriting history or weakening that guarantee for every other
5112    /// caller. A review-only run costs nothing extra — no implementation, no
5113    /// judging, no vote — and exercises the exact same review → verify →
5114    /// gate → (human) merge path, unmodified.
5115    pub async fn fix_selected(
5116        &mut self,
5117        ids: &[String],
5118        reason: &str,
5119        allow_stale: bool,
5120    ) -> Result<()> {
5121        let reason = reason.trim();
5122        if reason.is_empty() {
5123            bail!("a fix request needs a reason — that is the operator's own record of why");
5124        }
5125        if ids.is_empty() {
5126            bail!("no finding id given");
5127        }
5128        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5129            bail!(
5130                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5131                 has already concluded — can be given a targeted fix. A run still \
5132                 in progress should simply be resumed; a `merged` run's branch has \
5133                 already landed, so its answer is a fresh `magi review <branch>`, \
5134                 not reopening this run's own record",
5135                self.state.id,
5136                self.state.status.as_str()
5137            );
5138        }
5139        let Some(winner) = self.state.winner().cloned() else {
5140            bail!("run {} has no winning candidate to fix", self.state.id);
5141        };
5142        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5143            bail!(
5144                "branch `{}` no longer exists; this run cannot be extended",
5145                winner.branch
5146            );
5147        }
5148        let home = crate::run::home();
5149        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5150            bail!(
5151                "run {} is currently being worked on by another magi process",
5152                self.state.id
5153            );
5154        }
5155        // Held for the rest of this call, including the follow-up review
5156        // below: two `magi fix` invocations against the same run must not
5157        // both reach the worktree manipulation further down, which would
5158        // otherwise race to remove and recreate the same directory — see
5159        // [`FixClaim`]'s own doc.
5160        let _claim = FixClaim::acquire(&self.state.dir())?;
5161
5162        // Resolve every id before spending anything — an unknown id refuses
5163        // the whole request rather than silently dropping it — and dedup
5164        // while keeping the operator's own order.
5165        let mut seen = BTreeSet::new();
5166        let mut findings = Vec::new();
5167        let mut missing = Vec::new();
5168        for id in ids {
5169            if !seen.insert(id.clone()) {
5170                continue;
5171            }
5172            match self.state.finding(id) {
5173                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5174                    id: f.id.clone(),
5175                    severity: f.severity,
5176                    reviewer_vote: rec.vote,
5177                    round: round.round,
5178                    round_head: round.head.clone(),
5179                    reviewer: rec.reviewer,
5180                    agent: rec.agent.clone(),
5181                    file: f.file.clone(),
5182                    line: f.line,
5183                    title: f.title.clone(),
5184                    detail: f.detail.clone(),
5185                    outcome: OperatorFixOutcome::Pending,
5186                }),
5187                None => missing.push(id.clone()),
5188            }
5189        }
5190        if !missing.is_empty() {
5191            bail!(
5192                "unknown finding id(s): {}; nothing was changed",
5193                missing.join(", ")
5194            );
5195        }
5196
5197        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5198        let stale_details: Vec<(String, String)> = findings
5199            .iter()
5200            .filter(|f| f.round_head != head_at_request)
5201            .map(|f| (f.id.clone(), f.round_head.clone()))
5202            .collect();
5203        let stale = !stale_details.is_empty();
5204        if stale && !allow_stale {
5205            bail!(
5206                "the branch has moved since some finding(s) were raised — {} — now \
5207                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5208                stale_details
5209                    .iter()
5210                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5211                    .collect::<Vec<_>>()
5212                    .join(", "),
5213                short(&head_at_request)
5214            );
5215        }
5216
5217        let request = OperatorFixRequest {
5218            requested_at: Timestamp::now(),
5219            reason: reason.to_owned(),
5220            findings,
5221            head_at_request: head_at_request.clone(),
5222            allow_stale,
5223            stale,
5224            fix: None,
5225            result_head: None,
5226            follow_up_review_run: None,
5227        };
5228        self.state.event(
5229            "fix",
5230            format!(
5231                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5232                request.findings.len(),
5233                request
5234                    .findings
5235                    .iter()
5236                    .map(|f| f.id.as_str())
5237                    .collect::<Vec<_>>()
5238                    .join(", "),
5239            ),
5240        );
5241        // Recorded now, before any worktree work or the fixer call itself —
5242        // and re-saved at each checkpoint below: a crash at any point after
5243        // this (mid fixer call, mid follow-up review) must not lose the fact
5244        // that this was requested, for which findings, and why. Everything
5245        // past this point reads and writes through `request_index` rather
5246        // than a local variable, since `request` itself is moved here.
5247        self.state.operator_fixes.push(request);
5248        self.state.save()?;
5249        let request_index = self.state.operator_fixes.len() - 1;
5250
5251        // A fresh, dedicated worktree for this one call, never the winner's
5252        // own worktree in place: that one may already be gone (folded away),
5253        // and reusing it in place would leave the branch checked out there
5254        // when the follow-up review below tries to check it out again. Freed
5255        // immediately after, either way — but only once confirmed clean:
5256        // `worktree_remove` is a `git worktree remove --force`, which would
5257        // otherwise discard uncommitted work left there by the operator or
5258        // another process before this had a chance to even look at it.
5259        let repo = self.state.repo.clone();
5260        let coordination = self.coordinate(&repo).await?;
5261        if winner.worktree.exists() {
5262            // Lockfiles a rescue commit withheld stay untracked on purpose and
5263            // are already recorded; they are not the operator's work to protect.
5264            let dirty = git::git(
5265                &winner.worktree,
5266                &["status", "--porcelain", "--untracked-files=all"],
5267            )
5268            .await?;
5269            let only_withheld = dirty.lines().all(|l| {
5270                l.strip_prefix("?? ")
5271                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5272            });
5273            if !only_withheld {
5274                bail!(
5275                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5276                     discard them first",
5277                    winner.worktree.display()
5278                );
5279            }
5280            git::worktree_remove(&self.state.repo, &winner.worktree)
5281                .await
5282                .ok();
5283        }
5284        let fix_worktree = self.state.worktree_root().join("operator-fix");
5285        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5286        git::git(
5287            &self.state.repo,
5288            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5289        )
5290        .await
5291        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5292        if !git::is_clean(&fix_worktree).await? {
5293            git::worktree_remove(&self.state.repo, &fix_worktree)
5294                .await
5295                .ok();
5296            bail!(
5297                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5298                winner.branch
5299            );
5300        }
5301        let repo_for_setup = self.state.repo.clone();
5302        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5303            Ok(()) => {
5304                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5305            }
5306            Err(e) => Err(e),
5307        };
5308        if let Err(e) = setup {
5309            git::worktree_remove(&self.state.repo, &fix_worktree)
5310                .await
5311                .ok();
5312            return Err(e);
5313        }
5314
5315        let run_id = self.state.id.clone();
5316        let prompts = self.state.config.prompts.clone();
5317        let language = self.state.config.graph.language.clone();
5318        let sessions = self.state.config.graph.sessions;
5319        let artifacts = agent::artifacts_dir(&self.state.dir());
5320        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5321            .findings
5322            .iter()
5323            .map(|f| Finding {
5324                id: f.id.clone(),
5325                severity: f.severity,
5326                file: f.file.clone(),
5327                line: f.line,
5328                title: f.title.clone(),
5329                detail: f.detail.clone(),
5330            })
5331            .collect();
5332        let fix_prompt = prompt::operator_fix_in(
5333            &self.state.instruction,
5334            &finding_list,
5335            reason,
5336            &stale_details,
5337            &head_at_request,
5338            &language,
5339            &self.state.config.graph.github_language,
5340        );
5341        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5342        drop(coordination);
5343        let (job, seat, out) = self
5344            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5345                prompt: fix_prompt.clone(),
5346                spec,
5347                seat,
5348                cwd: fix_worktree.clone(),
5349                timeout,
5350                allow_write: true,
5351                sessions,
5352                artifacts: artifacts.clone(),
5353                stem: "operator-fix".to_owned(),
5354                handover: None,
5355            })
5356            .await;
5357        let agent_id = seat.agent.clone();
5358
5359        let mut fix = FixRecord {
5360            agent: agent_id,
5361            addressed: Vec::new(),
5362            rejected: Vec::new(),
5363            notes: String::new(),
5364            committed: false,
5365            failed: None,
5366            duration_ms: 0,
5367            continuation: None,
5368        };
5369        let mut final_seat = seat.clone();
5370        match out {
5371            AgentOutcome::Ok(o) => {
5372                fix.duration_ms = o.duration_ms;
5373                let parsed = verdict::extract_json::<FixReport>(&o.text);
5374                let incomplete_reason = match &parsed {
5375                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5376                        "the reply parsed, but it reported a command whose own CLI \
5377                         never confirmed an exit status"
5378                            .to_owned(),
5379                    ),
5380                    Ok(_) => None,
5381                    Err(e) => Some(e.to_string()),
5382                };
5383                match incomplete_reason {
5384                    None => {
5385                        let report = parsed.expect("checked Ok above");
5386                        fix.addressed = report.addressed;
5387                        fix.rejected = report.rejected;
5388                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5389                    }
5390                    Some(reason) => {
5391                        let (resumed_seat, resolved, failure, cont) = self
5392                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5393                            .await;
5394                        fix.duration_ms += cont.cumulative_wait_ms;
5395                        fix.continuation = Some(cont);
5396                        final_seat = resumed_seat;
5397                        match resolved {
5398                            Some(report) => {
5399                                fix.addressed = report.addressed;
5400                                fix.rejected = report.rejected;
5401                                fix.notes =
5402                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5403                            }
5404                            None => fix.failed = failure,
5405                        }
5406                    }
5407                }
5408            }
5409            AgentOutcome::Dropped(o) => {
5410                fix.duration_ms = o.duration_ms;
5411                let why = o
5412                    .dropped
5413                    .as_ref()
5414                    .map(|d| d.why.as_str())
5415                    .unwrap_or("the CLI ended the stream without delivering its answer");
5416                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5417            }
5418            AgentOutcome::Quota(o) => {
5419                self.state.quota.push(QuotaLoss {
5420                    seat: final_seat.key.clone(),
5421                    node: "fix".to_owned(),
5422                    at: Timestamp::now(),
5423                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5424                });
5425                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5426            }
5427            AgentOutcome::Failed(e) => fix.failed = Some(e),
5428            AgentOutcome::ResourceBlocked(e) => {
5429                fix.failed = Some(format!("the fixer was not started: {e}"));
5430            }
5431        }
5432        if fix.continuation.is_none() {
5433            fix.continuation = Some(ContinuationRecord::not_needed());
5434        }
5435        self.state.seats.insert(final_seat.key.clone(), final_seat);
5436
5437        let rescue_message = format!(
5438            "magi: operator-selected fix ({}) (uncommitted work)",
5439            self.state.operator_fixes[request_index]
5440                .findings
5441                .iter()
5442                .map(|f| f.id.as_str())
5443                .collect::<Vec<_>>()
5444                .join(", ")
5445        );
5446        let repo = self.state.repo.clone();
5447        let coordination = self.coordinate(&repo).await?;
5448        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5449            self.state.note_withheld("fix", &r.withheld);
5450        }
5451        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5452        fix.committed = after != head_at_request;
5453        git::worktree_remove(&self.state.repo, &fix_worktree)
5454            .await
5455            .ok();
5456        drop(coordination);
5457
5458        self.state.event(
5459            "fix",
5460            match &fix.failed {
5461                Some(reason) => format!(
5462                    "operator fix: adoption report was lost ({reason}); {}",
5463                    if fix.committed {
5464                        "committed"
5465                    } else {
5466                        "NO new commit"
5467                    }
5468                ),
5469                None => format!(
5470                    "operator fix: {} addressed, {} rejected, {}",
5471                    fix.addressed.len(),
5472                    fix.rejected.len(),
5473                    if fix.committed {
5474                        "committed"
5475                    } else {
5476                        "NO new commit"
5477                    }
5478                ),
5479            },
5480        );
5481
5482        // Every selected finding gets an outcome — never left `Pending` once
5483        // the fixer's own turn is over. A report that never came back at all
5484        // marks every one of them `Unreported`, not silently "not addressed":
5485        // quota, a dropped stream, or an exhausted continuation are gaps in
5486        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5487        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5488        for f in &mut self.state.operator_fixes[request_index].findings {
5489            f.outcome = if fix.failed.is_some() {
5490                OperatorFixOutcome::Unreported
5491            } else if fix.addressed.contains(&f.id) {
5492                OperatorFixOutcome::Addressed
5493            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5494                OperatorFixOutcome::Rejected { why: r.why.clone() }
5495            } else {
5496                OperatorFixOutcome::Unreported
5497            };
5498        }
5499
5500        let committed = fix.committed;
5501        if committed {
5502            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5503        }
5504        self.state.operator_fixes[request_index].fix = Some(fix);
5505        // Saved again now that the fixer's own outcome is final, on top of
5506        // the save right after the request was first pushed above.
5507        self.state.save()?;
5508
5509        if committed {
5510            self.state.event(
5511                "fix",
5512                format!(
5513                    "operator fix committed {}; opening a follow-up review-only run",
5514                    short(&after)
5515                ),
5516            );
5517            // The operator asked for the fix, and the follow-up serves whatever
5518            // task the run it follows served.
5519            let origin =
5520                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5521            match Self::review(
5522                &self.state.repo,
5523                &winner.branch,
5524                self.state.config.clone(),
5525                origin,
5526            )
5527            .await
5528            {
5529                Ok(mut follow_up) => {
5530                    follow_up.state.event(
5531                        "start",
5532                        format!(
5533                            "requested by an operator fix on run {} for finding(s) {}",
5534                            self.state.id,
5535                            self.state.operator_fixes[request_index]
5536                                .findings
5537                                .iter()
5538                                .map(|f| f.id.as_str())
5539                                .collect::<Vec<_>>()
5540                                .join(", "),
5541                        ),
5542                    );
5543                    follow_up.state.save()?;
5544                    let follow_up_id = follow_up.state.id.clone();
5545                    // The follow-up is a run like any other: it belongs to the
5546                    // task of the run it follows, or to one filed for it.
5547                    let adopted = match crate::direct::adopt(
5548                        &follow_up.state,
5549                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5550                    ) {
5551                        Ok(a) => a,
5552                        Err(e) => {
5553                            // An ownerless run must not spend agent calls; it
5554                            // stays saved, and `magi run --resume` adopts it.
5555                            self.state.event(
5556                                "fix",
5557                                format!(
5558                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5559                                ),
5560                            );
5561                            self.state.save()?;
5562                            return Ok(());
5563                        }
5564                    };
5565                    let executed = follow_up.execute().await;
5566                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5567                    if let Some(a) = adopted {
5568                        a.finish(&follow_up.state, executed);
5569                    }
5570                    if let Some(e) = failure {
5571                        self.state.event(
5572                            "fix",
5573                            format!(
5574                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5575                            ),
5576                        );
5577                    }
5578                    self.state.operator_fixes[request_index].follow_up_review_run =
5579                        Some(follow_up_id);
5580                }
5581                Err(e) => {
5582                    self.state.event(
5583                        "fix",
5584                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5585                    );
5586                }
5587            }
5588            self.state.save()?;
5589        }
5590
5591        Ok(())
5592    }
5593
5594    // --------------------------------------------------------------- review
5595
5596    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5597    /// names, in order, each at most once, moving on only when the call
5598    /// advances (an error, a quota hit or nothing usable - the same decision
5599    /// point `agent::chain_advances` is for the other chained roles).
5600    ///
5601    /// Each agent gets its own seat from `Runner::seat`: the same agent
5602    /// continues its conversation (the winner's own implementer seat when it
5603    /// is the winner's author, now that the competition is over), another
5604    /// takes a fresh one so the full prompt is sent again. A handover is
5605    /// recorded under `node`, which is what makes the fallback stick for the
5606    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5607    /// is returned, so an exhausted chain reads exactly like a single failed
5608    /// fixer: one quota loss, the same wording, the same refund. An earlier
5609    /// attempt's edits are left in the tree and judged, with the final
5610    /// attempt's, by what git says afterwards.
5611    ///
5612    /// Returns the job that produced the outcome, for `continue_fix_report`.
5613    async fn ask_fixer(
5614        &mut self,
5615        winner: &Candidate,
5616        node: &'static str,
5617        round: Option<usize>,
5618        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5619    ) -> (SeatJob, SeatState, AgentOutcome) {
5620        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5621        self.ask_fixer_among(attempts, node, round, None, build)
5622            .await
5623    }
5624
5625    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5626    /// worktree and a commit: before the next agent is asked after one that
5627    /// did not satisfy the call, the worktree is put back on that commit, so
5628    /// a failed agent's partial edits are not carried into the next one.
5629    async fn ask_fixer_among(
5630        &mut self,
5631        attempts: Vec<(AgentSpec, String)>,
5632        node: &'static str,
5633        round: Option<usize>,
5634        restore: Option<(&Path, &str)>,
5635        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5636    ) -> (SeatJob, SeatState, AgentOutcome) {
5637        let run_id = self.state.id.clone();
5638        let prompts = self.state.config.prompts.clone();
5639        let cache = self.state.config.cache_dir();
5640        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5641        let mut last = None;
5642        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5643            let seat = self.seat(&key, &spec.id);
5644            let mut job = build(spec.clone(), seat);
5645            if i > 0 {
5646                job.stem = format!("{}-{}", job.stem, spec.id);
5647            }
5648            let ctx = WaveCtx {
5649                carry_seats: false,
5650                pauses: Some((&self.pause, &self.interrupt)),
5651                run: &run_id,
5652                node,
5653                prompts: &prompts,
5654                cache: cache.as_deref(),
5655                round,
5656            };
5657            let (seat, out) =
5658                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5659            // A resource block never advances the chain: the chosen agent
5660            // was not asked, so it must be the one asked again on resume.
5661            let advances = match &out {
5662                AgentOutcome::Ok(o) => agent::output_advances(o),
5663                AgentOutcome::ResourceBlocked(_) => false,
5664                _ => true,
5665            };
5666            if let Some(next) = ids.get(i + 1)
5667                && advances
5668            {
5669                self.state.seats.insert(seat.key.clone(), seat.clone());
5670                let class =
5671                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5672                let reason = match &out {
5673                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5674                    other => fail_reason(other),
5675                };
5676                // A failed agent's edits must not reach the next one, and a
5677                // restore that did not work ends the chain: the call fails
5678                // and the caller's own (confirmed) restore decides.
5679                if let Some((wt, rev)) = restore
5680                    && let Err(e) = self.graft_restore(wt, rev).await
5681                {
5682                    let why =
5683                        format!("could not restore the worktree before the next fixer: {e:#}");
5684                    tracing::warn!("{why}");
5685                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5686                        AgentOutcome::ResourceBlocked(why)
5687                    } else {
5688                        AgentOutcome::Failed(why)
5689                    };
5690                    last = Some((job, seat, out));
5691                    break;
5692                }
5693                record_handover(
5694                    &mut self.state,
5695                    node,
5696                    &seat.key,
5697                    &spec.id,
5698                    next,
5699                    &class,
5700                    &reason,
5701                );
5702                continue;
5703            }
5704            last = Some((job, seat, out));
5705            break;
5706        }
5707        last.expect("the fixer chain always has an entry")
5708    }
5709
5710    /// The one repair a verification-only failure earns (see
5711    /// [`crate::recovery`]), before the first review round of a run that took
5712    /// a branch over.
5713    ///
5714    /// Saved as started *before* the fixer is called, so a resume never
5715    /// repairs twice; a start found on re-entry is rolled back to the commit
5716    /// it began from. A failed or unusable fixer leaves the tree as it was and
5717    /// the review goes on over it, so the first round's own verification
5718    /// reports the failure again. The failure is never carried into a round as
5719    /// a result: verification is re-run on whatever commit this leaves.
5720    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5721        use crate::recovery::{Mode, Repair};
5722        if !self.state.reviews.is_empty() {
5723            return Ok(());
5724        }
5725        let Some(mut rec) = self.state.recovery.clone() else {
5726            return Ok(());
5727        };
5728        if rec.mode != Mode::VerifyRepair {
5729            return Ok(());
5730        }
5731        let wt = winner.worktree.clone();
5732        let mut head = git::rev_parse(&wt, "HEAD").await?;
5733        match rec.repair.clone() {
5734            Repair::Done { .. } => return Ok(()),
5735            Repair::WaitingForRestore { before } => {
5736                self.graft_restore(&wt, &before).await?;
5737                head = before;
5738                rec.repair = Repair::NotStarted;
5739                self.state.recovery = Some(rec.clone());
5740                self.state.save()?;
5741            }
5742            Repair::Started { before } => {
5743                // Interrupted: the unfinished edits are not trusted.
5744                // An unconfirmed rollback blocks the run with `Started` kept,
5745                // so a resume retries it; the tree is never reviewed as is.
5746                if let Err(e) = self.graft_restore(&wt, &before).await {
5747                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5748                        return Err(e);
5749                    }
5750                    return self
5751                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5752                        .await;
5753                }
5754                rec.repair = Repair::Done {
5755                    before: before.clone(),
5756                    after: before,
5757                    committed: false,
5758                    failed: Some("interrupted; rolled back".to_owned()),
5759                };
5760                self.state
5761                    .event("recovery", "the repair was interrupted and rolled back");
5762                self.state.recovery = Some(rec);
5763                self.state.save()?;
5764                return Ok(());
5765            }
5766            Repair::NotStarted => {}
5767        }
5768        if crate::recovery::recheck(&mut rec, &head) {
5769            self.state.event("recovery", rec.summary());
5770            self.state.recovery = Some(rec);
5771            self.state.save()?;
5772            return Ok(());
5773        }
5774        rec.repair = Repair::Started {
5775            before: head.clone(),
5776        };
5777        self.state.recovery = Some(rec.clone());
5778        self.state.save()?;
5779
5780        let language = self.state.config.graph.language.clone();
5781        let sessions = self.state.config.graph.sessions;
5782        let artifacts = agent::artifacts_dir(&self.state.dir());
5783        let fix_prompt = prompt::recovery_fix(
5784            &self.state.instruction,
5785            &rec,
5786            self.state.config.graph.review_rounds,
5787            &language,
5788        );
5789        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5790        self.state.event(
5791            "recovery",
5792            format!(
5793                "repair: the fixer starts from the failed verification on {}",
5794                short(&head)
5795            ),
5796        );
5797        let (_job, seat, out) = self
5798            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5799                prompt: fix_prompt.clone(),
5800                spec,
5801                seat,
5802                cwd: wt.clone(),
5803                timeout,
5804                allow_write: true,
5805                sessions,
5806                artifacts: artifacts.clone(),
5807                stem: "fix-recovery".to_owned(),
5808                handover: None,
5809            })
5810            .await;
5811        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5812            rec.repair = Repair::WaitingForRestore { before: head };
5813            self.state.recovery = Some(rec);
5814            return self.stop_for_resource();
5815        }
5816        let failed = match &out {
5817            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5818            AgentOutcome::Quota(o) => {
5819                self.state.quota.push(QuotaLoss {
5820                    seat: seat.key.clone(),
5821                    node: "fix".to_owned(),
5822                    at: Timestamp::now(),
5823                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5824                });
5825                Some("rate limited (quota); the fixer could not run".to_owned())
5826            }
5827            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5828            AgentOutcome::Failed(e) => Some(e.clone()),
5829        };
5830        self.state.seats.insert(seat.key.clone(), seat);
5831        let coordination = self.coordinate(&wt).await?;
5832        let failed = match failed {
5833            // The agent's partial edits are not kept.
5834            Some(why) => {
5835                if let Err(e) = git::restore_to(&wt, &head).await {
5836                    return self
5837                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5838                        .await;
5839                }
5840                Some(why)
5841            }
5842            None => {
5843                if let Ok(r) = git::rescue_commit(
5844                    &wt,
5845                    "magi: recovery repair of the failed verification (uncommitted work)",
5846                )
5847                .await
5848                {
5849                    self.state.note_withheld("fix", &r.withheld);
5850                }
5851                None
5852            }
5853        };
5854        let after = git::rev_parse(&wt, "HEAD").await?;
5855        drop(coordination);
5856        self.state.event(
5857            "recovery",
5858            match &failed {
5859                Some(why) => format!("repair: nothing kept ({why})"),
5860                None if after != head => {
5861                    format!("repair: committed {}", short(&after))
5862                }
5863                None => "repair: the fixer made no commit".to_owned(),
5864            },
5865        );
5866        rec.repair = Repair::Done {
5867            before: head.clone(),
5868            committed: after != head,
5869            after,
5870            failed,
5871        };
5872        self.state.recovery = Some(rec);
5873        self.state.save()?;
5874        Ok(())
5875    }
5876
5877    /// A repair whose rollback could not be confirmed: the run is blocked and
5878    /// resumable, and the unconfirmed state is never reviewed.
5879    async fn stop_recovery(
5880        &mut self,
5881        mut rec: crate::recovery::RecoveryRecord,
5882        head: &str,
5883        why: String,
5884    ) -> Result<()> {
5885        // `Started` stays recorded: a resume retries the restore.
5886        rec.repair = crate::recovery::Repair::Started {
5887            before: head.to_owned(),
5888        };
5889        self.state.recovery = Some(rec);
5890        self.state
5891            .event("recovery", format!("repair could not be undone: {why}"));
5892        self.state.status = RunStatus::Blocked;
5893        self.state.save()?;
5894        bail!("recovery repair could not be rolled back: {why}")
5895    }
5896
5897    async fn review_loop(&mut self) -> Result<()> {
5898        loop {
5899            if self
5900                .state
5901                .base_sync
5902                .as_ref()
5903                .is_some_and(|s| s.conflict.is_some())
5904            {
5905                return Ok(());
5906            }
5907            if self.catchup_repairable() {
5908                let record = self.state.reviews.last().expect("repairable round").clone();
5909                let winner = self
5910                    .state
5911                    .winner()
5912                    .cloned()
5913                    .context("catch-up repair has no winner")?;
5914                if !self.fix_review_round(&winner, record).await? {
5915                    return Ok(());
5916                }
5917            }
5918            self.review_loop_pass().await?;
5919            if !self.catchup_repairable() {
5920                return Ok(());
5921            }
5922        }
5923    }
5924
5925    /// Derive an owed catch-up repair from durable round evidence, including
5926    /// reentry after a crash before the fixer was dispatched.
5927    fn catchup_repairable(&self) -> bool {
5928        self.state.reviews.len() < self.state.config.graph.review_rounds
5929            && self.state.reviews.last().is_some_and(|r| {
5930                !r.clean
5931                    && r.blocking == 0
5932                    && r.fix.is_none()
5933                    && r.e2e_status() == E2eStatus::Failed
5934            })
5935    }
5936
5937    async fn review_loop_pass(&mut self) -> Result<()> {
5938        // A base that would not rebase is a person's decision, not a review
5939        // round: nothing here would change the answer, and reviewers and a
5940        // fixer would be spending real budget on a tree that cannot land
5941        // regardless of what they find.
5942        if self
5943            .state
5944            .base_sync
5945            .as_ref()
5946            .is_some_and(|s| s.conflict.is_some())
5947        {
5948            return Ok(());
5949        }
5950        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5951        // agent files with `magi task add` name the run that paid for it. The
5952        // prompt overlay is cloned alongside it because the waves borrow it
5953        // while `self` is mutably borrowed by the node's own bookkeeping.
5954        let run_id = self.state.id.clone();
5955        let prompts = self.state.config.prompts.clone();
5956        let Some(winner) = self.state.winner().cloned() else {
5957            return Ok(());
5958        };
5959        let max_rounds = self.state.config.graph.review_rounds;
5960        // A clean round, an exhausted round budget, or a stalled tree (see
5961        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5962        // they are recorded — recomputed here, not read off `status`, so a
5963        // reentry into a run that already stopped restates the identical
5964        // verdict instead of silently handing back whatever an earlier node
5965        // in this same walk clobbered `status` to (a solo-candidate
5966        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5967        // below runs an empty range once the budget is spent, and would
5968        // otherwise fall through without touching `status` at all.
5969        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5970            // A reentry after a crash between the last round's save and
5971            // `stop_reviewing` reaches the hand-off here, not there.
5972            if status == RunStatus::Gating {
5973                self.record_contested_handoff();
5974            }
5975            self.state.status = status;
5976            self.state.save()?;
5977            return Ok(());
5978        }
5979        self.state.status = RunStatus::Reviewing;
5980        // A last recorded round whose own verification never resolved
5981        // (`ResourceBlocked` — the shared build cache, not the patch) is
5982        // never a concluded round, whatever the round budget says: starting
5983        // a fresh round on top of it would spend a whole new reviewer wave
5984        // re-reading an unchanged patch instead of just retrying the one
5985        // check that actually needs it, and once the budget is spent the
5986        // loop below has nothing left to do at all (its range is empty).
5987        // Retry that check directly instead, exactly the same retry
5988        // `stop_reviewing` already does for its own catch-up case.
5989        if self.state.reviews.last().is_some_and(|r| {
5990            r.e2e_status() == E2eStatus::ResourceBlocked
5991                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5992        }) {
5993            let shell = self.state.config.shell();
5994            return self
5995                .stop_unresolved_review(
5996                    "the last round's own verification never resolved",
5997                    &shell,
5998                    &winner.worktree,
5999                )
6000                .await;
6001        }
6002
6003        self.recovery_repair(&winner).await?;
6004        if self.resource_halt()? {
6005            return Ok(());
6006        }
6007
6008        let repo = self.state.repo.clone();
6009        let root = self.state.worktree_root();
6010        let language = self.state.config.graph.language.clone();
6011        let sessions = self.state.config.graph.sessions;
6012        let artifacts = self.merge_repair_artifacts();
6013        let base = self.landing_base();
6014        let base_short = short(&base);
6015        let reviewers = self.roles.reviewers.clone();
6016        let shell = self.state.config.shell();
6017
6018        for round in (self.state.reviews.len() + 1)..=max_rounds {
6019            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6020            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6021            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6022            // The prior round's own record, already persisted — never a
6023            // hand-carried variable of just its failing output: that is
6024            // exactly what let a round's e2e result drift out of sync with
6025            // which commit it was actually about (see `SCHEMA`'s doc for
6026            // schema 8). Judged against `head`, the commit reviewers are
6027            // about to look at now, so the summary always reads as "an
6028            // earlier head" here — this round's own patch has not been
6029            // checked yet.
6030            let prev_verification = self
6031                .state
6032                .reviews
6033                .last()
6034                .and_then(|r| r.verification_summary(&head));
6035            // Only the first round of a run that took a branch over: later
6036            // rounds have this run's own history.
6037            let prior_section = match (&self.state.recovery, round) {
6038                (Some(rec), 1) if self.state.reviews.is_empty() => {
6039                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6040                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6041                            Some(crate::recovery::head_of(
6042                                &git::diff(&winner.worktree, reviewed, "HEAD")
6043                                    .await
6044                                    .unwrap_or_default(),
6045                                crate::recovery::SINCE_DIFF_MAX,
6046                            ))
6047                        }
6048                        _ => None,
6049                    };
6050                    Some(prompt::recovery_review(rec, since.as_deref()))
6051                }
6052                _ => None,
6053            };
6054
6055            // Each reviewer gets its own detached checkout of exactly this
6056            // commit: nobody can perturb the winner's tree, and the fixer can
6057            // keep working without racing a reviewer.
6058            let mut jobs = Vec::new();
6059            let coordination = self.coordinate(&repo).await?;
6060            for (r, spec) in reviewers.iter().cloned().enumerate() {
6061                let wt = root.join(format!("review-{}", r + 1));
6062                // `clean -fdx` in a reset wipes setup's products, so every
6063                // round sets the seat up again.
6064                if wt.exists() {
6065                    git::reset_detached(&wt, &head).await?;
6066                } else {
6067                    git::worktree_add_detached(&repo, &wt, &head).await?;
6068                }
6069                self.setup_seat_worktree(&repo, &wt, false).await?;
6070                let seat_key = format!("review-{}", r + 1);
6071                // The seat starts the round on whoever answered it last, not
6072                // on the agent the spec names, so a failure is not re-paid.
6073                let spec = pick_start_spec(
6074                    &self.roles.reviewer_roster,
6075                    spec,
6076                    self.state.seat_history.get(&seat_key),
6077                );
6078                let seat = self.seat(&seat_key, &spec.id);
6079                jobs.push(SeatJob {
6080                    prompt: prompt::review(&prompt::ReviewCtx {
6081                        instruction: &self.state.instruction,
6082                        branch: &winner.branch,
6083                        base_short: &base_short,
6084                        stat: &stat,
6085                        patch: &patch,
6086                        verification: prev_verification.as_ref(),
6087                        prior: prior_section.as_deref(),
6088                        reviewers: reviewers.len(),
6089                        round,
6090                        rounds: max_rounds,
6091                        // A review-only run has no rankings, so nothing
6092                        // competed for this patch and the reviewer is told so.
6093                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6094                        lens: Lens::for_seat(r),
6095                        language: &language,
6096                        github_language: &self.state.config.graph.github_language,
6097                    }),
6098                    spec,
6099                    seat,
6100                    cwd: wt,
6101                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6102                    allow_write: false,
6103                    sessions,
6104                    artifacts: artifacts.clone(),
6105                    stem: format!("review-{round}-{}", r + 1),
6106                    handover: None,
6107                });
6108            }
6109
6110            drop(coordination);
6111            self.state.event(
6112                "review",
6113                format!(
6114                    "round {round}: {} reviewers on {}",
6115                    jobs.len(),
6116                    short(&head)
6117                ),
6118            );
6119            let mut quota_losses = Vec::new();
6120            let review_retries = self.state.config.graph.retries;
6121            let review_cache = self.state.config.cache_dir();
6122            let ctx = WaveCtx {
6123                carry_seats: true,
6124                pauses: Some((&self.pause, &self.interrupt)),
6125                run: &run_id,
6126                node: "review",
6127                prompts: &prompts,
6128                cache: review_cache.as_deref(),
6129                round: Some(round),
6130            };
6131            let results = ask_json_wave::<Review>(
6132                jobs,
6133                Arc::clone(&self.sem),
6134                review_retries,
6135                &self.roles.reviewer_roster,
6136                &ctx,
6137                &mut quota_losses,
6138                &mut self.state,
6139                &|_: &Review| Ok(()),
6140            )
6141            .await;
6142            if self.state.resource_block.is_some() {
6143                return self.stop_for_resource();
6144            }
6145            // Counted before the move below: how many of *this* round's
6146            // reviewer seats were lost to their own rate limit, as opposed to
6147            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6148            let round_quota_missing = quota_losses.len();
6149            self.state.quota.extend(quota_losses);
6150
6151            let mut records = Vec::new();
6152            let mut all_findings = Vec::new();
6153            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6154                let agent_id = seat.agent.clone();
6155                self.state.seats.insert(seat.key.clone(), seat);
6156                let mut record = ReviewRecord {
6157                    reviewer: r + 1,
6158                    agent: agent_id,
6159                    summary: String::new(),
6160                    findings: Vec::new(),
6161                    vote: None,
6162                    failed: None,
6163                    duration_ms: 0,
6164                    // Set for both outcomes: `failed: Some(_)` with
6165                    // `attempts > 0` is a seat every retry still lost, not a
6166                    // recovered one — only `failed: None` with `attempts > 0`
6167                    // reads as "answered after a nudge" (see this field's own
6168                    // doc).
6169                    attempts,
6170                };
6171                match res {
6172                    Ok((review, out)) => {
6173                        // Sanitized here, at the point every other piece of
6174                        // agent prose in this file is (candidate summaries,
6175                        // deliberation turns, vote reasons): a reviewer's own
6176                        // words are the one thing about it that could name
6177                        // it, and reconsideration below broadcasts this same
6178                        // summary and these same findings to every other
6179                        // seat on the panel.
6180                        record.summary =
6181                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6182                        record.vote = Some(review.vote);
6183                        record.duration_ms = out.duration_ms;
6184                        for (n, mut f) in review.findings.into_iter().enumerate() {
6185                            // ids are magi's, never the agent's: the fixer's
6186                            // adoption report is keyed by them.
6187                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6188                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6189                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6190                            // `file` is agent-supplied prose too, never
6191                            // checked against the real tree — the same
6192                            // exposure `title`/`detail` above have, just in
6193                            // a field easy to forget because it looks like a
6194                            // path rather than free text.
6195                            f.file = f
6196                                .file
6197                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6198                            all_findings.push(f.clone());
6199                            record.findings.push(f);
6200                        }
6201                        self.state.event(
6202                            "review",
6203                            format!(
6204                                "round {round}: reviewer {} voted {} with {} finding(s)",
6205                                r + 1,
6206                                review.vote.label(),
6207                                record.findings.len()
6208                            ),
6209                        );
6210                    }
6211                    Err(e) => {
6212                        record.failed = Some(e.to_string());
6213                        self.state.event(
6214                            "review",
6215                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6216                        );
6217                    }
6218                }
6219                records.push(record);
6220            }
6221
6222            // Tally the round's votes and, if they split, spend the one
6223            // round of reconsideration the split -> deliberate -> revote
6224            // shape `judge`/`vote` use for the panel, sized down to what a
6225            // read-only review round can afford: one round, and a revote
6226            // rather than an argument, because the panel already wrote its
6227            // reasoning down as findings the first time around.
6228            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6229            let vote_split =
6230                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6231            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6232            if vote_split {
6233                self.state.event(
6234                    "review",
6235                    format!(
6236                        "round {round}: votes split ({}) — one round of reconsideration",
6237                        initial_votes
6238                            .iter()
6239                            .map(|v| v.label())
6240                            .collect::<Vec<_>>()
6241                            .join(", ")
6242                    ),
6243                );
6244                // Seats read every seat's findings and votes, still numbered
6245                // and never named — the same anonymity `review` itself keeps.
6246                let panel: Vec<ReviewSeatReport<'_>> = records
6247                    .iter()
6248                    .filter_map(|r| {
6249                        r.vote.map(|vote| ReviewSeatReport {
6250                            reviewer: r.reviewer,
6251                            vote,
6252                            summary: &r.summary,
6253                            findings: &r.findings,
6254                        })
6255                    })
6256                    .collect();
6257
6258                let mut jobs = Vec::new();
6259                let mut seats_at = Vec::new();
6260                for (r, spec) in reviewers.iter().cloned().enumerate() {
6261                    // A seat with no initial vote has nothing to reconsider
6262                    // from and stays absent, the same as it stayed absent
6263                    // from `panel` above.
6264                    if records[r].vote.is_none() {
6265                        continue;
6266                    }
6267                    let wt = root.join(format!("review-{}", r + 1));
6268                    let seat_key = format!("review-{}", r + 1);
6269                    let spec = self.occupant(&seat_key, spec);
6270                    let seat = self.seat(&seat_key, &spec.id);
6271                    // A seat with no live session has already forgotten the
6272                    // initial review's prompt — restate the patch it is
6273                    // voting on, the same as `deliberate`/`vote` do for a
6274                    // judge in the same position.
6275                    // The panel already carries this seat's own review and
6276                    // vote, so restating the patch makes the prompt whole for
6277                    // a seat handed to another agent.
6278                    let build = |with_patch: bool| {
6279                        prompt::review_reconsider(&ReviewReconsiderCtx {
6280                            instruction: &self.state.instruction,
6281                            reviewer: r + 1,
6282                            lens: Lens::for_seat(r),
6283                            panel: &panel,
6284                            patch: with_patch.then_some(ReviewPatch {
6285                                branch: &winner.branch,
6286                                base_short: &base_short,
6287                                stat: &stat,
6288                                patch: &patch,
6289                            }),
6290                            round,
6291                            rounds: max_rounds,
6292                            language: &language,
6293                        })
6294                    };
6295                    let full = build(true);
6296                    let prompt = if has_context(&spec, &seat, sessions) {
6297                        build(false)
6298                    } else {
6299                        full.clone()
6300                    };
6301                    jobs.push(SeatJob {
6302                        prompt,
6303                        spec,
6304                        seat,
6305                        cwd: wt,
6306                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6307                        allow_write: false,
6308                        sessions,
6309                        artifacts: artifacts.clone(),
6310                        stem: format!("review-{round}-reconsider-{}", r + 1),
6311                        handover: Some(full),
6312                    });
6313                    seats_at.push(r);
6314                }
6315
6316                let mut recon_quota_losses = Vec::new();
6317                let recon_cache = self.state.config.cache_dir();
6318                let recon_ctx = WaveCtx {
6319                    carry_seats: true,
6320                    pauses: Some((&self.pause, &self.interrupt)),
6321                    run: &run_id,
6322                    node: "review",
6323                    prompts: &prompts,
6324                    cache: recon_cache.as_deref(),
6325                    round: Some(round),
6326                };
6327                let recon_results = ask_json_wave::<ReviewRevote>(
6328                    jobs,
6329                    Arc::clone(&self.sem),
6330                    review_retries,
6331                    &self.roles.reviewer_roster,
6332                    &recon_ctx,
6333                    &mut recon_quota_losses,
6334                    &mut self.state,
6335                    &|_: &ReviewRevote| Ok(()),
6336                )
6337                .await;
6338                if self.state.resource_block.is_some() {
6339                    return self.stop_for_resource();
6340                }
6341                self.state.quota.extend(recon_quota_losses);
6342
6343                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6344                    let agent_id = seat.agent.clone();
6345                    self.state.seats.insert(seat.key.clone(), seat);
6346                    let mut rec = ReviewRevoteRecord {
6347                        reviewer: r + 1,
6348                        agent: agent_id,
6349                        vote: None,
6350                        reason: String::new(),
6351                        failed: None,
6352                    };
6353                    match res {
6354                        Ok((rv, _)) => {
6355                            rec.vote = Some(rv.vote);
6356                            rec.reason =
6357                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6358                            self.state.event(
6359                                "review",
6360                                format!(
6361                                    "round {round}: reviewer {} revoted {}",
6362                                    r + 1,
6363                                    rv.vote.label()
6364                                ),
6365                            );
6366                        }
6367                        Err(e) => {
6368                            rec.failed = Some(e.to_string());
6369                            self.state.event(
6370                                "review",
6371                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6372                            );
6373                        }
6374                    }
6375                    reconsideration.push(rec);
6376                }
6377            } else if initial_votes.len() > 1 {
6378                self.state.event(
6379                    "review",
6380                    format!(
6381                        "round {round}: votes agreed ({}) — no reconsideration",
6382                        initial_votes[0].label()
6383                    ),
6384                );
6385            }
6386
6387            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6388            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6389            // A round that already has a blocking finding and a round left to
6390            // try is going back to the fixer no matter what `verify.e2e`
6391            // says, so running it first only spends the loop's slowest step
6392            // (minutes, for a Rust repo's full test suite) on a head about
6393            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6394            // runs once a round has no blocking findings left (see
6395            // `round_is_clean`, which a deferred — empty — `e2e` can never
6396            // satisfy since `blocking` is nonzero whenever this branch is
6397            // taken), and `stop_reviewing` forces a real run before it will
6398            // ever read a deferred round as green.
6399            let defer_e2e =
6400                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6401            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6402                let reason =
6403                    format!("{blocking} blocking finding(s) already required a fix this round");
6404                self.state.event(
6405                    "verify",
6406                    format!(
6407                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6408                         {}); it will run once a round has none left",
6409                        short(&head)
6410                    ),
6411                );
6412                (Vec::new(), false, true, Some(reason))
6413            } else {
6414                let e2e_commands = self.state.config.verify.e2e.clone();
6415                let cache_dir = self.state.config.cache_dir();
6416                let context = format!("round {round}");
6417                let wait = self.repository_wait();
6418                let (e2e, verify_retried) = with_cache_lease(
6419                    &mut self.state,
6420                    cache_dir.as_deref(),
6421                    "e2e",
6422                    "e2e",
6423                    &winner.worktree,
6424                    &head,
6425                    verify_timeout,
6426                    &context,
6427                    &wait,
6428                    |state, budget| {
6429                        let shell = shell.clone();
6430                        let e2e_commands = e2e_commands.clone();
6431                        let worktree = winner.worktree.clone();
6432                        let context = context.clone();
6433                        async move {
6434                            run_e2e_with_retry(
6435                                state,
6436                                &shell,
6437                                &e2e_commands,
6438                                &worktree,
6439                                budget,
6440                                &context,
6441                            )
6442                            .await
6443                        }
6444                    },
6445                )
6446                .await?;
6447                (e2e, verify_retried, false, None)
6448            };
6449
6450            let expected = records.len();
6451            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6452            let incomplete = answered < expected;
6453            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6454            let policy = self.state.config.graph.incomplete_review;
6455            let clean = round_is_clean(
6456                blocking,
6457                e2e_ok,
6458                answered,
6459                expected,
6460                round_quota_missing,
6461                policy,
6462            );
6463
6464            let mut round_record = ReviewRound {
6465                round,
6466                head: head.clone(),
6467                verified_head: None,
6468                verified_at: None,
6469                reviews: records,
6470                e2e,
6471                verify_retried,
6472                e2e_deferred,
6473                e2e_defer_reason,
6474                fix: None,
6475                blocking,
6476                answered,
6477                expected,
6478                clean,
6479                progressed: false,
6480                vote_split,
6481                reconsideration,
6482                verdict: None,
6483            };
6484            // The final vote per seat is its revote where reconsideration
6485            // ran and answered, its initial vote otherwise — the same
6486            // fallback `tally` uses for a judge whose private vote failed.
6487            round_record.verdict = ReviewVote::worst(
6488                round_record
6489                    .final_votes()
6490                    .into_iter()
6491                    .map(|(_, _, vote)| vote),
6492            );
6493            // Which commit and when magi actually attempted to check —
6494            // known the moment a command was dispatched against `head`,
6495            // whether or not it finished: a resource-blocked attempt still
6496            // targeted a specific commit at a specific time, and leaving
6497            // that unrecorded is exactly what made `verification_summary`
6498            // report a fresh attempt as "commit unknown ... recorded before
6499            // this was tracked", indistinguishable from a genuinely old,
6500            // untracked record. Only a deferred or unconfigured round never
6501            // ran at all and has nothing to record — see
6502            // `ReviewRound::verified_head`'s own doc.
6503            if !matches!(
6504                round_record.e2e_status(),
6505                E2eStatus::Deferred | E2eStatus::NotConfigured
6506            ) {
6507                round_record.verified_head = Some(head.clone());
6508                round_record.verified_at = Some(Timestamp::now());
6509            }
6510
6511            if incomplete {
6512                let missing: Vec<String> = round_record
6513                    .reviews
6514                    .iter()
6515                    .filter(|r| r.failed.is_some())
6516                    .map(|r| format!("review-{}", r.reviewer))
6517                    .collect();
6518                self.state.event(
6519                    "review",
6520                    format!(
6521                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6522                        missing.join(", ")
6523                    ),
6524                );
6525            }
6526
6527            if clean {
6528                self.state.event(
6529                    "review",
6530                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6531                        format!(
6532                            "round {round}: clean (warn policy, incomplete panel) — no \
6533                             blocking findings from the seats that answered, verification green"
6534                        )
6535                    } else if incomplete {
6536                        format!(
6537                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6538                             quorum) — no blocking findings from the seats that answered, \
6539                             verification green",
6540                            expected - answered
6541                        )
6542                    } else {
6543                        format!("round {round}: clean — no blocking findings, verification green")
6544                    },
6545                );
6546                self.state.reviews.push(round_record);
6547                self.state.status = RunStatus::Gating;
6548                self.state.save()?;
6549                return Ok(());
6550            }
6551
6552            // Nothing was raised and verification passed, but not every seat
6553            // answered and `round_is_clean` still refused to call it clean —
6554            // either a seat is missing for a reason other than its own quota
6555            // (a crash, a timeout, unparsable output — worth another try), or
6556            // every seat that could have answered lost its quota and nobody
6557            // is left to decide on: re-review rather than send the fixer
6558            // after a round with nothing to fix.
6559            if incomplete && blocking == 0 && e2e_ok {
6560                self.state.reviews.push(round_record);
6561                self.state.save()?;
6562                if round == max_rounds {
6563                    self.state.status = RunStatus::Blocked;
6564                    self.state.event(
6565                        "review",
6566                        format!(
6567                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6568                             refusing to call it clean",
6569                            expected - answered
6570                        ),
6571                    );
6572                    return Ok(());
6573                }
6574                continue;
6575            }
6576
6577            // Nothing for the fixer to act on (`blocking == 0`) and the only
6578            // reason this round is not clean is that magi itself never got
6579            // a command to run — the shared build cache, not the patch (see
6580            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6581            // the fixer would invite a change to appease contention that has
6582            // nothing to do with the diff, and would leave this attempt
6583            // sitting in the next round's prompt as if it were about an
6584            // earlier, superseded commit rather than what it actually is:
6585            // the same head, still waiting to be checked. Wait for it the
6586            // same way the final round's own contention is already handled,
6587            // whatever round this happens to be.
6588            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6589                self.state.reviews.push(round_record);
6590                return self
6591                    .stop_unresolved_review(
6592                        "the round's own verification could not run",
6593                        &shell,
6594                        &winner.worktree,
6595                    )
6596                    .await;
6597            }
6598
6599            if round == max_rounds {
6600                self.state.reviews.push(round_record);
6601                return self
6602                    .stop_reviewing(
6603                        &format!(
6604                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6605                        ),
6606                        &shell,
6607                        &winner.worktree,
6608                    )
6609                    .await;
6610            }
6611
6612            if !self.fix_review_round(&winner, round_record).await? {
6613                return Ok(());
6614            }
6615        }
6616        Ok(())
6617    }
6618
6619    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6620    /// False means admission or continuation policy stopped the review loop.
6621    async fn fix_review_round(
6622        &mut self,
6623        winner: &Candidate,
6624        mut round_record: ReviewRound,
6625    ) -> Result<bool> {
6626        let round = round_record.round;
6627        let shell = self.state.config.shell();
6628        let step = self.run_fix_round(winner, &mut round_record).await?;
6629        if matches!(step, FixStep::Resource) {
6630            self.stop_for_resource()?;
6631            return Ok(false);
6632        }
6633        if self.state.reviews.len() < round {
6634            self.state.reviews.push(round_record);
6635        } else {
6636            self.state.reviews[round - 1] = round_record;
6637        }
6638        self.state.save()?;
6639
6640        // The fixer's own report never came back this round, even after
6641        // `continue_fix_report`'s own budget was spent on it — not an
6642        // ordinary "no report" (dropped stream, quota, plain failure),
6643        // which already reads that way and is left to the existing round
6644        // budget. Stopping here, rather than opening another round, is
6645        // what keeps a next reviewer/fixer wave from ever being
6646        // dispatched onto `winner.worktree` while whatever the seat's
6647        // last call may still have running there is unaccounted for: no
6648        // process liveness check exists (and none is being added — see
6649        // AGENTS.md/this task's own scope), so the only way to honour
6650        // "nothing starts before a valid report returns" is to not start
6651        // anything further on this worktree from this run at all.
6652        if matches!(step, FixStep::Recorded { report_lost: true }) {
6653            self.stop_reviewing(
6654                "the fixer's adoption report never came back, even after resuming its \
6655                         own seat; refusing to start another round against the same worktree \
6656                         while that is unresolved",
6657                &shell,
6658                &winner.worktree,
6659            )
6660            .await?;
6661            return Ok(false);
6662        }
6663
6664        let streak = self
6665            .state
6666            .reviews
6667            .iter()
6668            .rev()
6669            .take_while(|r| !r.progressed)
6670            .count();
6671        if streak >= STAGNANT_LIMIT {
6672            self.stop_reviewing(
6673                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6674                &shell,
6675                &winner.worktree,
6676            )
6677            .await?;
6678            return Ok(false);
6679        }
6680        Ok(true)
6681    }
6682
6683    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6684    async fn run_fix_round(
6685        &mut self,
6686        winner: &Candidate,
6687        round_record: &mut ReviewRound,
6688    ) -> Result<FixStep> {
6689        let round = round_record.round;
6690        let max_rounds = self.state.config.graph.review_rounds;
6691        let run_id = self.state.id.clone();
6692        let prompts = self.state.config.prompts.clone();
6693        let language = self.state.config.graph.language.clone();
6694        let sessions = self.state.config.graph.sessions;
6695        let artifacts = self.merge_repair_artifacts();
6696        let base = self.landing_base();
6697        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6698        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6699        let this_round_verification = round_record.verification_summary(&head);
6700        let all_findings: Vec<_> = round_record
6701            .reviews
6702            .iter()
6703            .flat_map(|r| r.findings.iter().cloned())
6704            .collect();
6705        // Fix. The winner's own implementer seat continues its conversation:
6706        // the competition is over, so context is pure benefit now.
6707        let blocking_findings: Vec<_> = all_findings
6708            .iter()
6709            .filter(|f| f.severity.blocks())
6710            .cloned()
6711            .collect();
6712        let fix_prompt = prompt::fix_in(
6713            &self.state.instruction,
6714            &blocking_findings,
6715            this_round_verification.as_ref(),
6716            round,
6717            max_rounds,
6718            &language,
6719            &self.state.config.graph.github_language,
6720        );
6721        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6722        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6723        let (job, seat, out) = self
6724            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6725                prompt: fix_prompt.clone(),
6726                spec,
6727                seat,
6728                cwd: winner.worktree.clone(),
6729                timeout,
6730                allow_write: true,
6731                sessions,
6732                artifacts: artifacts.clone(),
6733                stem: format!("fix-{round}"),
6734                handover: None,
6735            })
6736            .await;
6737        // Not asked: the round is not recorded (so a resume repeats it),
6738        // no fixer was handed over to, and the seat is left as it was.
6739        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6740            return Ok(FixStep::Resource);
6741        }
6742        let agent_id = seat.agent.clone();
6743
6744        let mut fix = FixRecord {
6745            agent: agent_id,
6746            addressed: Vec::new(),
6747            rejected: Vec::new(),
6748            notes: String::new(),
6749            committed: false,
6750            failed: None,
6751            duration_ms: 0,
6752            continuation: None,
6753        };
6754        let mut continuation = ContinuationRecord::not_needed();
6755        let mut final_seat = seat.clone();
6756        match out {
6757            AgentOutcome::Ok(o) => {
6758                fix.duration_ms = o.duration_ms;
6759                let parsed = verdict::extract_json::<FixReport>(&o.text);
6760                // A parsed report standing next to a command this same
6761                // reply's own CLI never confirmed the exit status of is
6762                // not a resolved answer — the identical `CommandEvidence`
6763                // `state.jobs` renders, read here instead of only on
6764                // display, per the completion judgment and the shown
6765                // record needing to agree.
6766                let incomplete_reason = match &parsed {
6767                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6768                        "the reply parsed, but it reported a command whose own CLI never \
6769                             confirmed an exit status"
6770                            .to_owned(),
6771                    ),
6772                    Ok(_) => None,
6773                    Err(e) => Some(e.to_string()),
6774                };
6775                match incomplete_reason {
6776                    None => {
6777                        let report = parsed.expect("checked Ok above");
6778                        fix.addressed = report.addressed;
6779                        fix.rejected = report.rejected;
6780                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6781                    }
6782                    Some(reason) => {
6783                        let (resumed_seat, resolved, failure, cont) = self
6784                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6785                            .await;
6786                        fix.duration_ms += cont.cumulative_wait_ms;
6787                        continuation = cont;
6788                        final_seat = resumed_seat;
6789                        match resolved {
6790                            Some(report) => {
6791                                fix.addressed = report.addressed;
6792                                fix.rejected = report.rejected;
6793                                fix.notes =
6794                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6795                            }
6796                            None => fix.failed = failure,
6797                        }
6798                    }
6799                }
6800            }
6801            // The CLI's raw error JSON is not a fix report to parse.
6802            AgentOutcome::Dropped(o) => {
6803                fix.duration_ms = o.duration_ms;
6804                let why = o
6805                    .dropped
6806                    .as_ref()
6807                    .map(|d| d.why.as_str())
6808                    .unwrap_or("the CLI ended the stream without delivering its answer");
6809                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6810            }
6811            AgentOutcome::Quota(o) => {
6812                self.state.quota.push(QuotaLoss {
6813                    seat: final_seat.key.clone(),
6814                    node: "fix".to_owned(),
6815                    at: Timestamp::now(),
6816                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6817                });
6818                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6819            }
6820            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6821        }
6822        fix.continuation = Some(continuation);
6823        self.state.seats.insert(final_seat.key.clone(), final_seat);
6824        let coordination = self.coordinate(&winner.worktree).await?;
6825        if let Ok(r) = git::rescue_commit(
6826            &winner.worktree,
6827            &format!("magi: review round {round} fixes (uncommitted work)"),
6828        )
6829        .await
6830        {
6831            self.state.note_withheld("fix", &r.withheld);
6832        }
6833        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6834        fix.committed = after != before;
6835        // Judged by what `git` says moved against base, never by the
6836        // fixer's own `addressed`/`rejected` count — see
6837        // `ReviewRound::progressed`. Propagated with `?`, the same as the
6838        // `patch` snapshot above: swallowing this error would default
6839        // `diff_after` to empty, which almost always differs from a
6840        // non-empty `patch` and reads as "progressed" — exactly backwards
6841        // for a `git` failure the stagnation check cannot see through.
6842        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6843        drop(coordination);
6844        let progressed = diff_after != patch;
6845        let commit_note = if fix.committed {
6846            "committed"
6847        } else {
6848            "NO new commit"
6849        };
6850        let tree_note = if progressed {
6851            "changed vs base"
6852        } else {
6853            "unchanged vs base"
6854        };
6855        self.state.event(
6856            "fix",
6857            match &fix.failed {
6858                // Distinct on purpose from "0 addressed, 0 rejected": the
6859                // fixer's own diff still landed (blocking counts do keep
6860                // falling round over round), only its adoption report did
6861                // not come back, so this must never read like every
6862                // finding was reviewed and declined.
6863                Some(reason) => {
6864                    format!(
6865                        "round {round}: fixer's adoption report was lost ({reason}); \
6866                             {commit_note}, tree {tree_note}"
6867                    )
6868                }
6869                None => format!(
6870                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6871                         {tree_note}{}",
6872                    fix.addressed.len(),
6873                    fix.rejected.len(),
6874                    if continuation.outcome == ContinuationOutcome::Resumed {
6875                        format!(
6876                            " (adoption report recovered after {} continuation(s))",
6877                            continuation.attempts
6878                        )
6879                    } else {
6880                        String::new()
6881                    },
6882                ),
6883            },
6884        );
6885        round_record.fix = Some(fix);
6886        round_record.progressed = progressed;
6887        Ok(FixStep::Recorded {
6888            report_lost: matches!(
6889                continuation.outcome,
6890                ContinuationOutcome::Exhausted
6891                    | ContinuationOutcome::QuotaLost
6892                    | ContinuationOutcome::NoSession
6893            ),
6894        })
6895    }
6896
6897    /// Decide, from the last recorded round's own verification, whether
6898    /// stopping the review loop is a hand-off or a genuine block.
6899    ///
6900    /// Called once the loop has given up trying — the round budget is spent,
6901    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6902    /// findings still open, never while a round is still clean or the
6903    /// incomplete-panel case handled inline above. Gate and e2e are facts
6904    /// about the tree; a lingering review finding is an opinion, and this
6905    /// workload's own `magi stats` puts reviewer precision low enough
6906    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6907    /// must not by itself stand between a green, verified change and the
6908    /// human who decides what to do with it. A red e2e is not an opinion, so
6909    /// that case still blocks, with the failing command and a tail of its
6910    /// output recorded here rather than left in `run.json` for someone to go
6911    /// find.
6912    ///
6913    /// A round that deferred its own e2e (see [`Config::graph`]'s
6914    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6915    /// only because nothing ran, and treating an empty list as a passing one
6916    /// here is exactly the "deferred painted green" bug this function exists
6917    /// to not have. When the last round's own verification never resolved —
6918    /// deferred on purpose, or a real attempt the shared build cache blocked
6919    /// — this makes (or retries) the real run, on the actual worktree this
6920    /// loop is about to stop touching, before deciding anything. A
6921    /// resource-blocked attempt is likewise never read as either green or
6922    /// red: it is evidence about the machine, not the patch (see
6923    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6924    /// blocked cache leaves this call without deciding rather than guessing
6925    /// — the caller retries on a later reentry.
6926    /// Record, once, that the review loop handed off over a blocking finding
6927    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6928    /// `land` asks the owner even with `land_approval` off. Called from every
6929    /// path that concludes `Gating`; a reentry keeps the first record.
6930    fn record_contested_handoff(&mut self) {
6931        if self.state.contested_handoff.is_some() {
6932            return;
6933        }
6934        let Some(contested) = self
6935            .state
6936            .reviews
6937            .last()
6938            .and_then(ReviewRound::contested_handoff)
6939        else {
6940            return;
6941        };
6942        self.state.event(
6943            "review",
6944            format!(
6945                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6946                 wait for the owner's approval",
6947                contested.findings.len(),
6948                contested.rejecters.len()
6949            ),
6950        );
6951        self.state.contested_handoff = Some(contested);
6952    }
6953
6954    async fn stop_unresolved_review(
6955        &mut self,
6956        why: &str,
6957        shell: &[String],
6958        worktree: &Path,
6959    ) -> Result<()> {
6960        self.stop_reviewing_inner(why, shell, worktree, true).await
6961    }
6962
6963    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6964        self.stop_reviewing_inner(why, shell, worktree, false).await
6965    }
6966
6967    async fn stop_reviewing_inner(
6968        &mut self,
6969        why: &str,
6970        shell: &[String],
6971        worktree: &Path,
6972        repairable: bool,
6973    ) -> Result<()> {
6974        let round_idx = self.state.reviews.len() - 1;
6975        // A deferred round and a resource-blocked one are the same shape
6976        // here: neither has a real result yet, and both get one more
6977        // attempt. Read off `e2e_status` — the single source for this —
6978        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6979        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6980        // still retries instead of being read as a settled result the
6981        // instant it stops being empty.
6982        let needs_catchup_run = matches!(
6983            self.state.reviews[round_idx].e2e_status(),
6984            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6985        );
6986        if needs_catchup_run {
6987            let round = self.state.reviews[round_idx].round;
6988            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6989            let commands = self.state.config.verify.e2e.clone();
6990            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6991            let cache_dir = self.state.config.cache_dir();
6992            let context = format!(
6993                "round {round}: verification unresolved, catching up before the final decision"
6994            );
6995            let wait = self.repository_wait();
6996            let (outcomes, verify_retried) = with_cache_lease(
6997                &mut self.state,
6998                cache_dir.as_deref(),
6999                "e2e",
7000                "e2e",
7001                worktree,
7002                &attempted_head,
7003                timeout,
7004                &context,
7005                &wait,
7006                |state, budget| {
7007                    let shell = shell.to_vec();
7008                    let commands = commands.clone();
7009                    let context = context.clone();
7010                    async move {
7011                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7012                            .await
7013                    }
7014                },
7015            )
7016            .await?;
7017            let last = &mut self.state.reviews[round_idx];
7018            last.e2e = outcomes;
7019            last.verify_retried = verify_retried;
7020            // Always the commit and time this attempt actually targeted,
7021            // whether or not it happens to equal the reviewed `head` and
7022            // whether or not a command finished — see
7023            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7024            // attempt is recorded too, so a later reader sees "attempted
7025            // again at T2" rather than silence.
7026            last.verified_head = Some(attempted_head);
7027            last.verified_at = Some(Timestamp::now());
7028            if verify_inconclusive(&last.e2e) {
7029                // Still not a real result: `e2e_deferred` is left exactly
7030                // as it was, so `needs_catchup_run` above reads
7031                // `ResourceBlocked` (via `e2e_status`, which checks
7032                // `resource_blocked` before `e2e_deferred`) and retries
7033                // again on the next reentry, rather than recording
7034                // contention as a red e2e and blocking the run on it.
7035                self.state.save()?;
7036                return Ok(());
7037            }
7038            last.e2e_deferred = false;
7039        }
7040        let last = &self.state.reviews[round_idx];
7041        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7042
7043        match last.e2e_status() {
7044            E2eStatus::Failed => {
7045                let red: Vec<String> = last
7046                    .e2e
7047                    .iter()
7048                    .filter(|o| !o.ok())
7049                    .map(|o| {
7050                        format!(
7051                            "`{}` -> {:?}\n{}",
7052                            o.command,
7053                            o.code,
7054                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7055                        )
7056                    })
7057                    .collect();
7058                let can_repair = last.blocking == 0 && last.fix.is_none();
7059                self.state.reviews[round_idx].clean = false;
7060                self.state
7061                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7062                self.state.status = RunStatus::Blocked;
7063                if repairable
7064                    && can_repair
7065                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7066                {
7067                    self.state.status = RunStatus::Reviewing;
7068                }
7069            }
7070            // `needs_catchup_run` above already retried once this call; if
7071            // it is still blocked, this is magi's own admission it could
7072            // not get a command to run, never a verdict on the patch — the
7073            // run is left exactly where a later reentry can retry again.
7074            E2eStatus::ResourceBlocked => {
7075                self.state.event(
7076                    "review",
7077                    format!(
7078                        "{why}; e2e could not run (resource unavailable); not \
7079                         deciding yet"
7080                    ),
7081                );
7082            }
7083            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7084                self.state.event(
7085                    "review",
7086                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7087                );
7088                self.record_contested_handoff();
7089                self.state.status = RunStatus::Gating;
7090            }
7091        }
7092        self.state.save()?;
7093        Ok(())
7094    }
7095
7096    // ----------------------------------------------------------------- gate
7097
7098    async fn gate(&mut self) -> Result<()> {
7099        // Judged by the review record itself, not by `status`: a solo
7100        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7101        // reentry (see `judge`), and trusting it here is exactly how a run
7102        // that exhausted its review budget got gated and merged a second
7103        // time around. `review_conclusion` recomputes the review loop's own
7104        // verdict from the round records themselves — `Gating` for a clean
7105        // round or a hand-off (see `stop_reviewing`), anything else means the
7106        // loop is still going or genuinely blocked.
7107        // A base the winner could not be replayed onto is a decision, not a
7108        // round: there is no landing tree to gate. Read as its own record for
7109        // the same reason the review verdict is.
7110        if self.state.status == RunStatus::Failed
7111            || self
7112                .state
7113                .base_sync
7114                .as_ref()
7115                .is_some_and(|s| s.conflict.is_some())
7116            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7117                != Some(RunStatus::Gating)
7118        {
7119            return Ok(());
7120        }
7121        if self.state.gate_ran {
7122            // `review_loop` derives its conclusion from the clean review
7123            // record on every reentry and therefore puts a completed run back
7124            // in `Gating`. A recorded gate is a stronger, terminal fact:
7125            // retain its original command output (or lack of any, for a repo
7126            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7127            // doc) and restore `Blocked` on a real failure rather than
7128            // pretending the command is still running or running it a second
7129            // time. `gate_ran == false` remains the only shape — unattempted,
7130            // or a resource-blocked retry — that may still need to execute a
7131            // command.
7132            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7133                self.state.status = RunStatus::Blocked;
7134                self.state.save()?;
7135            }
7136            return Ok(());
7137        }
7138        let Some(winner) = self.state.winner().cloned() else {
7139            return Ok(());
7140        };
7141        self.state.status = RunStatus::Gating;
7142        let mut outcomes = self.run_gate(&winner).await?;
7143        loop {
7144            // A resource-blocked outcome means the gate command never actually
7145            // ran - the shared build cache could not be acquired or confirmed
7146            // fresh in time - which is evidence about the machine, not about
7147            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7148            // Recording it as a red gate would mark a run `Blocked` on nothing
7149            // but contention magi has already logged; leaving `self.state.gate`
7150            // empty and `self.state.gate_ran` false instead keeps the shape
7151            // this function already treats as "still needs to run" (see the
7152            // early-return above), so the next call retries the command
7153            // rather than concluding anything.
7154            if verify_inconclusive(&outcomes) {
7155                self.state.save()?;
7156                return Ok(());
7157            }
7158            if outcomes.iter().all(CommandOutcome::ok) {
7159                break;
7160            }
7161            match self.gate_fix_round(&winner, &outcomes).await? {
7162                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7163                GateFix::Stop => break,
7164                GateFix::Defer => {
7165                    self.state.save()?;
7166                    return Ok(());
7167                }
7168            }
7169        }
7170        let passed = outcomes.iter().all(CommandOutcome::ok);
7171        self.state.gate = outcomes;
7172        self.state.gate_ran = true;
7173        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7174        if !passed {
7175            self.state.status = RunStatus::Blocked;
7176            let spent = self.state.gate_fixes.len();
7177            self.state.event(
7178                "gate",
7179                if spent == 0 {
7180                    "gate failed; not merging".to_owned()
7181                } else {
7182                    format!("gate failed after {spent} gate-fix round(s); not merging")
7183                },
7184            );
7185        }
7186        self.state.save()?;
7187        Ok(())
7188    }
7189
7190    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7191    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7192    /// after review is clean and never on a candidate awaiting judging.
7193    ///
7194    /// Never fails the run: a non-zero exit or timeout is a warning and a
7195    /// recorded outcome, and the gate remains the single arbiter. Nothing
7196    /// configured means nothing happens - no event, no commit. `commit_all`
7197    /// commits any leftover change under the neutral identity and returns
7198    /// `false` when the tree is clean, so no empty commit is ever made.
7199    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7200        let commands = state.config.verify.pre_gate.clone();
7201        if commands.is_empty() {
7202            return;
7203        }
7204        let shell = state.config.shell();
7205        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7206        let (outcomes, _) = run_commands(
7207            state,
7208            "pre_gate",
7209            "pre_gate",
7210            0,
7211            &shell,
7212            &commands,
7213            &winner.worktree,
7214            timeout,
7215        )
7216        .await;
7217        for o in &outcomes {
7218            if !o.ok() {
7219                tracing::warn!(
7220                    "pre_gate `{}` failed ({:?}); the gate decides",
7221                    o.command,
7222                    o.code
7223                );
7224            }
7225            state.event(
7226                "pre_gate",
7227                format!(
7228                    "`{}` -> {}",
7229                    o.command,
7230                    if o.ok() {
7231                        "pass".to_owned()
7232                    } else {
7233                        format!(
7234                            "FAIL ({:?})\n{}",
7235                            o.code,
7236                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7237                        )
7238                    }
7239                ),
7240            );
7241        }
7242        state.pre_gate = outcomes;
7243        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7244            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7245                Ok(head) => {
7246                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7247                    state.pre_gate_commit = Some(head);
7248                }
7249                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7250            },
7251            Ok(false) => {}
7252            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7253        }
7254        if let Err(e) = state.save() {
7255            tracing::warn!("could not persist the pre_gate record: {e:#}");
7256        }
7257    }
7258
7259    /// Run `verify.gate` once against the winner's current tree, logging one
7260    /// event per command. Empty when nothing is configured.
7261    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7262        let wait = self.repository_wait();
7263        run_landing_gate(&mut self.state, winner, &wait).await
7264    }
7265
7266    /// One bounded fix round for a failing gate.
7267    ///
7268    /// The fixer is told the failure came from the gate itself, not from a
7269    /// reviewer, and is shown the failed commands, their exit codes and a tail
7270    /// of their output - whatever `[verify].gate` holds, nothing here knows
7271    /// what those commands run. Only a normal non-zero exit that printed
7272    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7273    /// command or a full disk says nothing about the code, and a fixer sent
7274    /// after it can only appease the machine. The round is judged by what git
7275    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7276    /// again before the gate does, so a fix cannot trade a green gate for a
7277    /// red e2e unnoticed.
7278    async fn gate_fix_round(
7279        &mut self,
7280        winner: &Candidate,
7281        outcomes: &[CommandOutcome],
7282    ) -> Result<GateFix> {
7283        let cap = self.state.config.graph.gate_fix_rounds;
7284        let spent = self.state.gate_fixes.len();
7285        if spent >= cap {
7286            if cap > 0 {
7287                self.state.event(
7288                    "gate",
7289                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7290                );
7291            }
7292            return Ok(GateFix::Stop);
7293        }
7294        if !gate_fixable(outcomes) {
7295            self.state.event(
7296                "gate",
7297                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7298                 command or similar); not spending a fix round on it",
7299            );
7300            return Ok(GateFix::Stop);
7301        }
7302        let min_free = self.state.config.disk.min_free_bytes;
7303        if min_free > 0 {
7304            match crate::disk::free_bytes(&winner.worktree) {
7305                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7306                Ok(free) => {
7307                    self.state.event(
7308                        "gate",
7309                        format!(
7310                            "only {free} bytes free ({min_free} required by `[disk] \
7311                             min_free_bytes`); not spending a fix round on a failure the disk \
7312                             may explain"
7313                        ),
7314                    );
7315                    return Ok(GateFix::Stop);
7316                }
7317                Err(e) => {
7318                    self.state.event(
7319                        "gate",
7320                        format!("free disk space could not be measured ({e:#}); no fix round"),
7321                    );
7322                    return Ok(GateFix::Stop);
7323                }
7324            }
7325        }
7326
7327        let attempt = spent + 1;
7328        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7329        let base = self.landing_base();
7330        let fix_prompt = prompt::gate_fix_in(
7331            &self.state.instruction,
7332            &failed,
7333            attempt,
7334            cap,
7335            &self.state.config.graph.language,
7336            &self.state.config.graph.github_language,
7337        );
7338        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7339        let sessions = self.state.config.graph.sessions;
7340        let artifacts = self.merge_repair_artifacts();
7341        self.state.event(
7342            "gate",
7343            format!("gate failed; gate-fix round {attempt} of {cap}"),
7344        );
7345        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7346        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7347        // Persist before invoking the fixer: it may commit and then be
7348        // interrupted before its result (or the following e2e) is saved.
7349        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7350            head: before.clone(),
7351            passed: false,
7352            outcomes: Vec::new(),
7353        });
7354        self.state.save()?;
7355        let (_, seat, out) = self
7356            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7357                prompt: fix_prompt.clone(),
7358                spec,
7359                seat,
7360                cwd: winner.worktree.clone(),
7361                timeout,
7362                allow_write: true,
7363                sessions,
7364                artifacts: artifacts.clone(),
7365                stem: format!("gate-fix-{attempt}"),
7366                handover: None,
7367            })
7368            .await;
7369        // The fixer was never asked: no round is spent (nothing is recorded)
7370        // and a later reentry asks the same agent again.
7371        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7372            return Ok(GateFix::Defer);
7373        }
7374        let mut record = GateFixRecord {
7375            agent: seat.agent.clone(),
7376            failed,
7377            notes: String::new(),
7378            committed: false,
7379            error: None,
7380        };
7381        match out {
7382            AgentOutcome::Ok(o) => {
7383                // A missing report is not a failed fix: the round is judged
7384                // by the tree below, and the report only carries prose.
7385                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7386                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7387                }
7388            }
7389            AgentOutcome::Dropped(_) => {
7390                record.error = Some("the CLI dropped the stream".to_owned());
7391            }
7392            AgentOutcome::Quota(o) => {
7393                self.state.quota.push(QuotaLoss {
7394                    seat: seat.key.clone(),
7395                    node: "gate-fix".to_owned(),
7396                    at: Timestamp::now(),
7397                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7398                });
7399                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7400            }
7401            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7402        }
7403        self.state.seats.insert(seat.key.clone(), seat);
7404        let coordination = self.coordinate(&winner.worktree).await?;
7405        if let Ok(r) = git::rescue_commit(
7406            &winner.worktree,
7407            &format!("magi: gate fix {attempt} (uncommitted work)"),
7408        )
7409        .await
7410        {
7411            self.state.note_withheld("gate-fix", &r.withheld);
7412        }
7413        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7414        record.committed = after != before;
7415        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7416        let note = record.error.clone();
7417        self.state.gate_fixes.push(record);
7418        self.state.save()?;
7419        if !changed {
7420            self.state.event(
7421                "gate",
7422                match note {
7423                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7424                    None => format!("gate-fix round {attempt}: the tree did not change"),
7425                },
7426            );
7427            return Ok(GateFix::Stop);
7428        }
7429        self.state.event(
7430            "gate",
7431            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7432        );
7433
7434        drop(coordination);
7435        let wait = self.repository_wait();
7436        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7437    }
7438
7439    // ---------------------------------------------------------------- merge
7440
7441    /// One read-only rewrite by the summary's author; a text the gate still
7442    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7443    /// applies on their word or on silence. `None` means the run is parked on
7444    /// that question and nothing may be pushed or opened yet.
7445    async fn guarded_pr_message(
7446        &mut self,
7447        winner: &Candidate,
7448        facts: Option<&BranchFacts>,
7449        posting: bool,
7450    ) -> Result<Option<PrMessage>> {
7451        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7452        let head = match final_head.as_deref() {
7453            Some(h) => crate::pr_outcome::FinalHead::At(h),
7454            None => crate::pr_outcome::FinalHead::Unknown,
7455        };
7456        // Only a pull request that is about to be posted pays for a summary.
7457        let summarized = if posting {
7458            Some(self.summarize_change(winner, final_head.as_deref()).await)
7459        } else {
7460            None
7461        };
7462        // A busy shared resource is not a permanent cause: publishing an
7463        // "unavailable" description would stick (a later head move never
7464        // edits the body). Stop before anything is pushed or opened; the
7465        // run is `Blocked` and a resume asks the summary again.
7466        if self.state.resource_block.is_some() {
7467            self.state.event(
7468                "pr-summary",
7469                "a shared resource was busy; nothing was published, resume to summarize again",
7470            );
7471            return Ok(None);
7472        }
7473        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7474            anyhow::bail!(
7475                "the final commit moved during public description generation; retry before publishing"
7476            );
7477        }
7478        let note = match (&summarized, &self.state.change_summary) {
7479            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7480            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7481            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7482            (None, _) => ChangeNote::NotAsked,
7483        };
7484        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7485        if !posting {
7486            let identity = crate::scrub::Identity::current();
7487            return Ok(Some(PrMessage {
7488                title: crate::scrub::scrub(&pr.title, &identity),
7489                body: crate::scrub::scrub(&pr.body, &identity),
7490            }));
7491        }
7492        // This very text was already rejected and asked about: no second
7493        // judge call, rewrite or question, just carry the record on.
7494        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7495        if let Some(known) = self
7496            .state
7497            .github_text
7498            .clone()
7499            .filter(|g| g.fingerprint == raw_fp)
7500        {
7501            return self.settle_withheld(winner, &pr, known).await;
7502        }
7503        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7504        let mut violations =
7505            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7506        let identity = crate::scrub::Identity::current();
7507        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7508            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7509            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7510        {
7511            violations.push(crate::github_text::Violation::SensitiveData);
7512        }
7513        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7514        // only a language violation starts one; sensitive data alone goes
7515        // straight to the owner, with the redacted text as the fallback.
7516        let has_language = violations
7517            .iter()
7518            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7519        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7520            self.state.event(
7521                "github-text",
7522                if has_language {
7523                    format!("description rejected: {violations:?}; requesting one rewrite")
7524                } else {
7525                    format!("description rejected: {violations:?}; asking the owner")
7526                },
7527            );
7528            let mut rewritten = false;
7529            let mut rejected_rewrite = None;
7530            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7531            {
7532                let output = text;
7533                if let Some(title) = summary_title(&output) {
7534                    // Judge the redacted candidate that will actually be posted.
7535                    // Sensitive data alone must not discard a useful rewrite.
7536                    let raw_body = summary_without_title(&output);
7537                    let clean_title = crate::scrub::scrub(&title, &identity);
7538                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7539                    let redacted = clean_title != title || body != raw_body;
7540                    let title = clean_title;
7541                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7542                    if !complete {
7543                        self.state.event(
7544                            "github-text",
7545                            "description rewrite omitted required findings",
7546                        );
7547                    }
7548                    if !body.trim().is_empty() {
7549                        body.push_str("\n\n");
7550                        body.push_str(&public_pr_evidence(&self.state, head));
7551                    }
7552                    let marker = format!("magi:run/{}", self.state.id);
7553                    if !body.trim().is_empty() && !body.contains(&marker) {
7554                        body.push_str(&format!("\n\n{marker}"));
7555                    }
7556                    let rewrite = PrMessage {
7557                        title: title.clone(),
7558                        body: body.clone(),
7559                    };
7560                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7561                    let mut failed = crate::github_text::check_for(
7562                        &self.state.config.graph,
7563                        &title,
7564                        &body,
7565                        redecision,
7566                    );
7567                    if (!crate::github_text::shareable(&title)
7568                        || !crate::github_text::shareable(&body))
7569                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7570                    {
7571                        failed.push(crate::github_text::Violation::SensitiveData);
7572                    }
7573                    if complete
7574                        && !title.trim().is_empty()
7575                        && !body.trim().is_empty()
7576                        && !failed.is_empty()
7577                    {
7578                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7579                            title: title.clone(),
7580                            body: body.clone(),
7581                            failed: failed
7582                                .iter()
7583                                .map(|v| crate::github_text::category(*v).to_owned())
7584                                .collect(),
7585                        });
7586                    }
7587                    if complete
7588                        && !title.trim().is_empty()
7589                        && !raw_body.trim().is_empty()
7590                        && failed.is_empty()
7591                    {
7592                        decision = redecision;
7593                        pr = PrMessage { title, body };
7594                        rewritten = true;
7595                        self.state
7596                            .event("github-text", "description rewrite passed");
7597                        if redacted {
7598                            self.state.event(
7599                                "github-text",
7600                                "sensitive data removed from description rewrite",
7601                            );
7602                        }
7603                    }
7604                }
7605            }
7606            if !rewritten {
7607                self.state.event(
7608                    "github-text",
7609                    "description rewrite unavailable or rejected; asking the owner",
7610                );
7611                if let Some(w) =
7612                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7613                {
7614                    // A different text than the one asked about before: that
7615                    // question no longer describes anything.
7616                    self.retire_github_text_question("the withheld text changed");
7617                    let record = crate::run::GithubTextAsk {
7618                        rewrite: rejected_rewrite,
7619                        fingerprint: raw_fp,
7620                        title: w.title,
7621                        body: w.body,
7622                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7623                        question: None,
7624                        asks: 0,
7625                        resolved: false,
7626                        chosen_title: None,
7627                        choice: None,
7628                    };
7629                    return self.settle_withheld(winner, &pr, record).await;
7630                }
7631            }
7632        }
7633        let (title, body) =
7634            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7635        Ok(Some(PrMessage { title, body }))
7636    }
7637
7638    /// Describe the whole change from its final diff in either run mode.
7639    ///
7640    /// The summary is tied to the final commit and to the merge base of the
7641    /// resolved remote base; a record for any other pair is dropped and the
7642    /// agent asked again, so a description never outlives the diff it was
7643    /// written from. The seat is read-only and the prompt tells it to read the
7644    /// diff and the code around it. `[roles] synthesizer` selects the agents
7645    /// (unset: the default pick), each asked at most once, advancing on an
7646    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7647    /// carries the reason it is unavailable; nothing is invented then.
7648    async fn summarize_change(
7649        &mut self,
7650        winner: &Candidate,
7651        head: Option<&str>,
7652    ) -> std::result::Result<(), String> {
7653        let Some(head) = head else {
7654            return Err("the final commit could not be read".to_owned());
7655        };
7656        let remote = self.state.config.merge.remote.clone();
7657        let tracking = format!("{remote}/{}", self.state.base_branch);
7658        if !git::rev_exists(&winner.worktree, &tracking).await {
7659            return Err(format!("the remote base `{tracking}` could not be read"));
7660        }
7661        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7662            Ok(b) if !b.is_empty() => b,
7663            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7664        };
7665        let evidence = crate::pr_outcome::summary_evidence(
7666            &self.state,
7667            crate::pr_outcome::FinalHead::At(head),
7668        );
7669        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7670        if self
7671            .state
7672            .change_summary
7673            .as_ref()
7674            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7675        {
7676            return Ok(());
7677        }
7678        if self.state.change_summary.take().is_some() {
7679            self.state.event(
7680                "pr-summary",
7681                "the final diff or evidence changed; the earlier change summary was discarded",
7682            );
7683        }
7684        // Save the source snapshot even if no generation agent is available.
7685        if let Err(e) = write_artifact(
7686            &self.state,
7687            &format!("public-description-{fingerprint}.md"),
7688            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7689        ) {
7690            tracing::warn!("could not save public description evidence: {e:#}");
7691        }
7692        let chain = match agent::pick_chain(
7693            &self.state.config.agents,
7694            self.state.config.roles.synthesizer.as_ref(),
7695            &agent::installed,
7696            "synthesizer",
7697        ) {
7698            Ok(chain) => chain,
7699            Err(e) => {
7700                tracing::warn!("no agent for the change summary: {e:#}");
7701                self.state.event("pr-summary", "no summary agent available");
7702                return Err("no agent was available to summarize it".to_owned());
7703            }
7704        };
7705        let subjects = git::subjects(&winner.worktree, &base, head)
7706            .await
7707            .unwrap_or_default();
7708        let stat = git::diff_stat(&winner.worktree, &base, head)
7709            .await
7710            .unwrap_or_default();
7711        let stat: String = stat.chars().take(6_000).collect();
7712        let mut prompt = crate::prompt::change_summary_in(
7713            &base,
7714            head,
7715            &subjects,
7716            stat.trim(),
7717            &self.state.config.graph.github_language,
7718        );
7719        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7720        let sessions = self.state.config.graph.sessions;
7721        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7722        let prompts = self.state.config.prompts.clone();
7723        let cache = self.state.config.cache_dir();
7724        let run = self.state.id.clone();
7725        for (i, spec) in chain.into_iter().enumerate() {
7726            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7727            let github_spec = match agent::for_github_text(
7728                &spec,
7729                &self.state.config.graph.github_language,
7730                &winner.worktree,
7731            ) {
7732                Ok(spec) => spec,
7733                Err(e) => {
7734                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7735                    continue;
7736                }
7737            };
7738            let job = SeatJob {
7739                spec: github_spec,
7740                seat,
7741                cwd: winner.worktree.clone(),
7742                prompt: prompt.clone(),
7743                timeout,
7744                allow_write: false,
7745                sessions,
7746                artifacts: agent::artifacts_dir(&self.state.dir()),
7747                stem: if i == 0 {
7748                    "change-summary".to_owned()
7749                } else {
7750                    format!("change-summary-{}", spec.id)
7751                },
7752                handover: None,
7753            };
7754            let ctx = WaveCtx {
7755                run: &run,
7756                node: "pr-summary",
7757                prompts: &prompts,
7758                cache: cache.as_deref(),
7759                round: None,
7760                carry_seats: false,
7761                pauses: Some((&self.pause, &self.interrupt)),
7762            };
7763            let (seat, outcome) =
7764                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7765            self.state.seats.insert(seat.key.clone(), seat);
7766            let text = match outcome {
7767                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7768                AgentOutcome::ResourceBlocked(why) => {
7769                    // No agent was asked: the run stops resumably before
7770                    // anything is published (see `guarded_pr_message`).
7771                    if self.state.resource_block.is_none() {
7772                        self.state.resource_block = Some(crate::run::ResourceBlock {
7773                            cause: crate::run::ResourceBlockCause::Other,
7774                            node: "pr-summary".to_owned(),
7775                            kind: "shared resource".to_owned(),
7776                            detail: why,
7777                            at: Timestamp::now(),
7778                        });
7779                    }
7780                    return Err("a shared resource was busy".to_owned());
7781                }
7782                _ => continue,
7783            };
7784            let blind = &self.state.config.blind;
7785            let Some(title) = summary_title(&text) else {
7786                continue;
7787            };
7788            let body: String = summary_without_title(&text)
7789                .lines()
7790                .filter(|l| !l.contains("magi:run/"))
7791                .collect::<Vec<_>>()
7792                .join("\n");
7793            let body = blind::sanitize_prose(body.trim(), blind);
7794            let title = blind::sanitize_prose(&title, blind);
7795            if body.trim().is_empty() || title.trim().is_empty() {
7796                continue;
7797            }
7798            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7799                continue;
7800            }
7801            // The branch must still be the one the agent was shown.
7802            if git::rev_parse(&winner.worktree, "HEAD")
7803                .await
7804                .ok()
7805                .as_deref()
7806                != Some(head)
7807            {
7808                return Err("the branch moved while the summary was written".to_owned());
7809            }
7810            if git::merge_base(&winner.worktree, &tracking, head)
7811                .await
7812                .ok()
7813                .as_deref()
7814                != Some(base.as_str())
7815                || crate::pr_outcome::summary_evidence(
7816                    &self.state,
7817                    crate::pr_outcome::FinalHead::At(head),
7818                ) != evidence
7819            {
7820                return Err("the summary evidence changed while it was written".to_owned());
7821            }
7822            self.state.change_summary = Some(crate::run::ChangeSummary {
7823                head: head.to_owned(),
7824                base,
7825                evidence: fingerprint,
7826                title,
7827                body,
7828            });
7829            self.state.event(
7830                "pr-summary",
7831                format!("change summary written by {} for the final diff", spec.id),
7832            );
7833            if let Err(e) = self.state.save() {
7834                tracing::warn!("could not save the change summary: {e:#}");
7835            }
7836            return Ok(());
7837        }
7838        self.state
7839            .event("pr-summary", "no agent produced a change summary");
7840        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7841    }
7842
7843    /// One read-only rewrite request for a withheld description. The
7844    /// implementer's own conversation is tried first when it exists and can be
7845    /// resumed; then the `[roles] language_judge` chain (or the default pick)
7846    /// writes it in a separate conversation, skipping the implementer's agent.
7847    /// The prompt carries the whole text, so no agent needs any memory. Each
7848    /// agent is asked at most once and the chain advances on an error, quota,
7849    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
7850    /// that has run out of quota must not end the rewrite. Nothing is recorded
7851    /// as a handover.
7852    async fn rewrite_pr_text(
7853        &mut self,
7854        winner: &Candidate,
7855        pr: &PrMessage,
7856        violations: &[crate::github_text::Violation],
7857    ) -> Option<String> {
7858        let prompt = format!(
7859            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7860            pr.title, pr.body
7861        );
7862        let sessions = self.state.config.graph.sessions;
7863        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7864        // (spec, the resumed implementer's seat). A fallback seat is created
7865        // only when its turn comes, so an unused fallback leaves no record.
7866        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7867        let key = format!("impl-{}", winner.label);
7868        if let Some(spec) = self
7869            .state
7870            .config
7871            .agents
7872            .iter()
7873            .find(|a| a.id == winner.agent)
7874            .cloned()
7875            && let Some(existing) = self.state.seats.get(&key)
7876            && existing.agent == spec.id
7877            && agent::has_session(spec.kind, existing, sessions)
7878        {
7879            attempts.push((spec, Some(existing.clone())));
7880        }
7881        match agent::pick_chain(
7882            &self.state.config.agents,
7883            self.state.config.roles.language_judge.as_ref(),
7884            &agent::installed,
7885            "language_judge",
7886        ) {
7887            Ok(chain) => {
7888                for spec in chain {
7889                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7890                        continue;
7891                    }
7892                    attempts.push((spec, None));
7893                }
7894            }
7895            Err(e) if attempts.is_empty() => {
7896                tracing::warn!("no agent for the description rewrite: {e:#}");
7897                self.state
7898                    .event("github-text", "no rewrite agent available");
7899                return None;
7900            }
7901            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7902        }
7903        let prompts = self.state.config.prompts.clone();
7904        let cache = self.state.config.cache_dir();
7905        let run = self.state.id.clone();
7906        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7907            let resumed = resumed_seat.is_some();
7908            let seat = resumed_seat.unwrap_or_else(|| {
7909                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7910            });
7911            let github_spec = match agent::for_github_text(
7912                &spec,
7913                &self.state.config.graph.github_language,
7914                &winner.worktree,
7915            ) {
7916                Ok(spec) => spec,
7917                Err(e) => {
7918                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7919                    continue;
7920                }
7921            };
7922            let job = SeatJob {
7923                spec: github_spec,
7924                seat,
7925                cwd: winner.worktree.clone(),
7926                prompt: prompt.clone(),
7927                timeout: retry_budget(timeout_full, resumed),
7928                allow_write: false,
7929                sessions,
7930                artifacts: agent::artifacts_dir(&self.state.dir()),
7931                stem: if i == 0 {
7932                    "github-text-rewrite".to_owned()
7933                } else {
7934                    format!("github-text-rewrite-{}", spec.id)
7935                },
7936                handover: None,
7937            };
7938            let ctx = WaveCtx {
7939                run: &run,
7940                node: "github-text",
7941                prompts: &prompts,
7942                cache: cache.as_deref(),
7943                round: None,
7944                carry_seats: false,
7945                pauses: Some((&self.pause, &self.interrupt)),
7946            };
7947            let (seat, outcome) =
7948                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7949            self.state.seats.insert(seat.key.clone(), seat);
7950            match outcome {
7951                AgentOutcome::Ok(o)
7952                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7953                {
7954                    return Some(o.text);
7955                }
7956                AgentOutcome::ResourceBlocked(_) => return None,
7957                _ => {}
7958            }
7959        }
7960        None
7961    }
7962
7963    /// Abandon the standing posting-gate question, if any.
7964    fn retire_github_text_question(&mut self, why: &str) {
7965        let Some(id) = self
7966            .state
7967            .github_text
7968            .as_ref()
7969            .and_then(|g| g.question.clone())
7970        else {
7971            return;
7972        };
7973        let _ = ask::Questions::open().update(&id, |q| {
7974            q.abandon(why);
7975            Ok(())
7976        });
7977    }
7978
7979    /// Carry a withheld text to its end: file the question, park while it is
7980    /// open, and on an answer, silence or timeout pick the text to post. The
7981    /// decision is saved before it is used, so it applies once.
7982    async fn settle_withheld(
7983        &mut self,
7984        winner: &Candidate,
7985        pr: &PrMessage,
7986        mut g: crate::run::GithubTextAsk,
7987    ) -> Result<Option<PrMessage>> {
7988        use crate::github_text as gt;
7989        let store = ask::Questions::open();
7990        let timeout = self.state.config.graph.answer_timeout;
7991        if !g.resolved {
7992            let mut standing = g
7993                .question
7994                .as_ref()
7995                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7996            if let Some(q) = standing.as_ref()
7997                && q.status.open()
7998                && gt::expired(q, timeout)
7999            {
8000                let why = format!("no answer within {}s of asking", timeout.max(1));
8001                standing = store
8002                    .update(&q.id, |q| {
8003                        q.abandon(&why);
8004                        Ok(())
8005                    })
8006                    .ok()
8007                    .map(|(q, ())| q);
8008            }
8009            let mut ask_again: Option<Option<Vec<String>>> = None;
8010            match standing {
8011                // Recorded but never written (a stop between the two saves),
8012                // or removed: ask again without spending the retry.
8013                None if g.asks <= 1 => {
8014                    g.asks = 0;
8015                    ask_again = Some(None);
8016                }
8017                None => {
8018                    g.resolved = true;
8019                    g.chosen_title = None;
8020                    g.choice = Some(GithubTextChoice::Fallback);
8021                }
8022                Some(q) if q.status.open() => {
8023                    self.park_on_github_text(&g)?;
8024                    return Ok(None);
8025                }
8026                Some(q) => match gt::read_reply(&q) {
8027                    gt::Reply::Fallback => {
8028                        g.resolved = true;
8029                        g.chosen_title = None;
8030                        g.choice = Some(GithubTextChoice::Fallback);
8031                    }
8032                    gt::Reply::Original => {
8033                        g.resolved = true;
8034                        g.chosen_title = None;
8035                        g.choice = Some(GithubTextChoice::Original);
8036                    }
8037                    gt::Reply::Rewrite => {
8038                        g.resolved = true;
8039                        g.chosen_title = None;
8040                        g.choice = Some(if g.rewrite.is_some() {
8041                            GithubTextChoice::Rewrite
8042                        } else {
8043                            GithubTextChoice::Fallback
8044                        });
8045                    }
8046                    gt::Reply::Title(text) => {
8047                        let first = text
8048                            .lines()
8049                            .map(str::trim)
8050                            .find(|l| !l.is_empty())
8051                            .unwrap_or("");
8052                        let decision = if first.is_empty() {
8053                            None
8054                        } else {
8055                            let candidate = PrMessage {
8056                                title: first.to_owned(),
8057                                body: String::new(),
8058                            };
8059                            self.judge_pr_language(&winner.worktree, &candidate).await
8060                        };
8061                        match gt::vet_title(
8062                            &text,
8063                            decision,
8064                            self.state.config.graph.github_english(),
8065                        ) {
8066                            Ok(title) => {
8067                                g.resolved = true;
8068                                g.chosen_title = Some(title);
8069                                g.choice = Some(GithubTextChoice::MyTitle);
8070                            }
8071                            Err(cats) if g.asks < 2 => {
8072                                ask_again = Some(Some(
8073                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8074                                ))
8075                            }
8076                            Err(_) => {
8077                                g.resolved = true;
8078                                g.chosen_title = None;
8079                                g.choice = Some(GithubTextChoice::Fallback);
8080                            }
8081                        }
8082                    }
8083                },
8084            }
8085            if let Some(rejected) = ask_again {
8086                // A retry (after a rejected replacement title) is still about
8087                // the original text: the same ORIGINAL / FALLBACK comparison.
8088                let w = gt::Withheld {
8089                    title: g.title,
8090                    body: g.body,
8091                    categories: g.categories.clone(),
8092                };
8093                // Named by the original text's fingerprint alone, so a resume
8094                // rewrites the same file and the detail stays identical.
8095                let artifact = match crate::run::write_artifact(
8096                    &self.state,
8097                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8098                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8099                ) {
8100                    Ok(p) => Some(p),
8101                    Err(e) => {
8102                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8103                        None
8104                    }
8105                };
8106                let (detail, comparison) = gt::question_content(
8107                    &w,
8108                    &pr.title,
8109                    &pr.body,
8110                    &self.state.id,
8111                    rejected.as_deref(),
8112                    artifact.as_deref(),
8113                    g.rewrite.as_ref(),
8114                );
8115                let mut q = ask::Question::new(
8116                    self.state.id.clone(),
8117                    gt::ASK_NODE.to_owned(),
8118                    gt::ASK_SEAT.to_owned(),
8119                    gt::question_summary(&self.state.config.graph.language, &w),
8120                    detail,
8121                    gt::question_choices(&w, g.rewrite.as_ref()),
8122                );
8123                q.posting_comparison = Some(comparison);
8124                q.answer_timeout = timeout;
8125                // Recorded before it is written: a stop in between leaves a
8126                // record naming a question the resume then files again.
8127                g.question = Some(q.id.clone());
8128                g.asks += 1;
8129                self.state.github_text = Some(g.clone());
8130                self.state.event(
8131                    "github-text",
8132                    format!("asking the owner about the withheld text ({})", q.short()),
8133                );
8134                self.state.save()?;
8135                store
8136                    .put_executable(&mut q)
8137                    .context("file the posting-gate question")?;
8138                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8139                    tracing::warn!(
8140                        "could not notify about posting-gate question {}: {e:#}",
8141                        q.short()
8142                    );
8143                }
8144                self.park_on_github_text(&g)?;
8145                return Ok(None);
8146            }
8147        }
8148        let resolved = g.chosen_title.clone();
8149        let original = g.choice == Some(GithubTextChoice::Original);
8150        // Field names only: the event log never records the text.
8151        self.state.event(
8152            "github-text",
8153            if original {
8154                format!(
8155                    "posting the original text for: {}",
8156                    match (g.title, g.body) {
8157                        (true, true) => "title, description",
8158                        (true, false) => "title",
8159                        _ => "description",
8160                    }
8161                )
8162            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8163                "posting the saved rewrite title and description".to_owned()
8164            } else if resolved.is_some() {
8165                "posting the owner's replacement title".to_owned()
8166            } else {
8167                "posting the neutral text for the withheld fields".to_owned()
8168            },
8169        );
8170        self.state.github_text = Some(g.clone());
8171        self.state.save()?;
8172        if g.choice == Some(GithubTextChoice::Rewrite)
8173            && let Some(r) = g.rewrite.as_ref()
8174        {
8175            return Ok(Some(PrMessage {
8176                title: r.title.clone(),
8177                body: r.body.clone(),
8178            }));
8179        }
8180        let w = gt::Withheld {
8181            title: g.title,
8182            body: g.body,
8183            categories: g.categories.clone(),
8184        };
8185        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8186        if let Some(t) = resolved {
8187            title = t;
8188        }
8189        // Every withheld field is now neutral or vetted English.
8190        let english = gt::LanguageDecision {
8191            title_english: true,
8192            body_english: true,
8193            body_complete: true,
8194        };
8195        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8196        if original {
8197            // The owner's explicit choice overrides the gate for the fields it
8198            // was made for; the others keep what the gate produced.
8199            if g.title {
8200                title = pr.title.clone();
8201            }
8202            if g.body {
8203                body = pr.body.clone();
8204            }
8205        }
8206        Ok(Some(PrMessage { title, body }))
8207    }
8208
8209    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8210        self.state.github_text = Some(g.clone());
8211        self.state.end_pass_parked(ParkCause::Question);
8212        self.state.parked = true;
8213        self.state.event(
8214            "github-text",
8215            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8216        );
8217        self.state.save()
8218    }
8219
8220    /// Ask `[roles] language_judge` about this text and record which source
8221    /// decided; the text itself is never recorded.
8222    async fn judge_pr_language(
8223        &mut self,
8224        cwd: &Path,
8225        pr: &PrMessage,
8226    ) -> Option<crate::github_text::LanguageDecision> {
8227        self.state.config.roles.language_judge.as_ref()?;
8228        if !self.state.config.graph.language_gate() {
8229            return None;
8230        }
8231        let decision =
8232            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8233        self.state.event(
8234            "github-text",
8235            if decision.is_some() {
8236                "language decided by the language judge"
8237            } else {
8238                "language judge unavailable; using built-in heuristics"
8239            },
8240        );
8241        decision
8242    }
8243
8244    async fn merge(&mut self) -> Result<()> {
8245        // Same reasoning as `gate`: ask the review and gate records directly
8246        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8247        // skip can rewrite on reentry to something that no longer says
8248        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8249        // so a hand-off (open findings, green verification) reaches merge
8250        // exactly like a genuinely clean round does.
8251        //
8252        // A run resumed mid-`land` never reaches here at all: `execute`
8253        // recognises `RunStatus::Landing` before it even calls `prep`, and
8254        // routes straight to `run_land` instead. That has to happen a level
8255        // up from this function, not with a check in here, because
8256        // `review_loop`'s own status recomputation (see its doc) runs
8257        // *before* `merge` on every reentry and would otherwise overwrite
8258        // the `Landing` marker with `Gating` before this node ever saw it.
8259        if self
8260            .state
8261            .base_sync
8262            .as_ref()
8263            .is_some_and(|s| s.conflict.is_some())
8264            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8265                != Some(RunStatus::Gating)
8266            // `gate_ran == false` is not "passed" - `gate` leaves it false
8267            // both before it has ever run and when its last attempt was
8268            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8269            // permission to merge on nothing but the review record. Only a
8270            // gate that actually ran may proceed; an intentional local
8271            // review can also finish with verification explicitly not configured.
8272            // `RunState::gate_status` computes that distinction.
8273            || !self.state.gate_allows_progress()
8274        {
8275            return Ok(());
8276        }
8277        // This node's own record, not `status`: `status == Ready` is not
8278        // unique to the harmless `MergeMode::None` path this line was
8279        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8280        // run's PR was closed without merging — and on that run `mode` is
8281        // still `Pr`, so a reentry that fell through here would push and
8282        // open a second pull request. `self.state.merge` is set exactly once
8283        // this node (or `land`) has already produced a verdict, under every
8284        // mode, which is what "already done" actually means here.
8285        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8286            return Ok(());
8287        }
8288        let Some(winner) = self.state.winner().cloned() else {
8289            return Ok(());
8290        };
8291        let repo = self.state.repo.clone();
8292        let base = self.state.base_branch.clone();
8293        let mode = self.state.config.merge.mode;
8294        let style = self.state.config.merge.style;
8295        let facts = if is_review_run(&self.state) {
8296            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8297            let start = review_base(
8298                &repo,
8299                &self.state.config.merge.remote,
8300                &base,
8301                &self.state.base_commit,
8302                &winner.branch,
8303            )
8304            .await;
8305            branch_facts(&repo, &start, &winner.branch).await
8306        } else {
8307            None
8308        };
8309        // `None` when the base could not be freshly read: then an adopted
8310        // pull request's title is left alone.
8311        let leaked = if is_review_run(&self.state) {
8312            leaked_subjects(&self.state, &winner.branch).await
8313        } else {
8314            Some(Vec::new())
8315        };
8316        // `None`: parked on the owner's word about a withheld title or body.
8317        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8318        // the resume arrives here again.
8319        let Some(pr) = self
8320            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8321            .await?
8322        else {
8323            return Ok(());
8324        };
8325        let message = pr.commit_message();
8326
8327        let outcome = match mode {
8328            MergeMode::None => MergeOutcome {
8329                mode,
8330                ok: true,
8331                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8332                empty: false,
8333            },
8334            MergeMode::Pr | MergeMode::Local
8335                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8336            {
8337                MergeOutcome {
8338                    mode,
8339                    ok: false,
8340                    detail: empty_candidate_detail(&self.state, &base),
8341                    empty: true,
8342                }
8343            }
8344            MergeMode::Local => {
8345                let on = git::current_branch(&repo).await?;
8346                if on.as_deref() != Some(base.as_str()) {
8347                    MergeOutcome {
8348                        mode,
8349                        ok: false,
8350                        detail: format!(
8351                            "{} has {} checked out, not the base branch {base}",
8352                            repo.display(),
8353                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8354                        ),
8355                        empty: false,
8356                    }
8357                } else if !git::is_clean(&repo).await? {
8358                    MergeOutcome {
8359                        mode,
8360                        ok: false,
8361                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8362                        empty: false,
8363                    }
8364                } else {
8365                    let out = match style {
8366                        MergeStyle::Merge => {
8367                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8368                        }
8369                        MergeStyle::Squash => {
8370                            git::merge_squash(&repo, &winner.branch, &message).await?
8371                        }
8372                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8373                    };
8374                    MergeOutcome {
8375                        mode,
8376                        ok: out.ok(),
8377                        detail: if out.ok() { out.stdout } else { out.stderr },
8378                        empty: false,
8379                    }
8380                }
8381            }
8382            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8383                self.state.pr_publication_pending = true;
8384                self.state.save()?;
8385                let out =
8386                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8387                self.state.pr_publication_pending = out.is_err();
8388                MergeOutcome {
8389                    mode,
8390                    ok: out.is_ok(),
8391                    detail: match out {
8392                        Ok(url) => url,
8393                        Err(e) => format!("{e:#}"),
8394                    },
8395                    empty: false,
8396                }
8397            }
8398            MergeMode::Pr => {
8399                let remote = self.state.config.merge.remote.clone();
8400                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8401                if !pushed.ok() {
8402                    MergeOutcome {
8403                        mode,
8404                        ok: false,
8405                        detail: pushed.stderr,
8406                        empty: false,
8407                    }
8408                } else {
8409                    // A retry or resume of a run whose branch already has an
8410                    // open pull request adopts it rather than failing on a
8411                    // duplicate. Only this winner branch into this base:
8412                    // `branch_for` derives the name from the run id, so a
8413                    // different run's pull request never matches.
8414                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8415                    let out = match pr_merge_plan(found) {
8416                        PrPlan::Create => {
8417                            gh_pr_create(
8418                                &winner.worktree,
8419                                &base,
8420                                &winner.branch,
8421                                &pr.title,
8422                                &pr.body,
8423                            )
8424                            .await
8425                        }
8426                        PrPlan::Adopt { url, title } => {
8427                            self.state
8428                                .event("merge", format!("Pr: adopted open pull request {url}"));
8429                            if self.state.github_text.as_ref().is_some_and(|g| {
8430                                g.resolved
8431                                    && g.choice == Some(GithubTextChoice::Rewrite)
8432                                    && g.rewrite.is_some()
8433                            }) {
8434                                // The owner chose the entire pair, even if the
8435                                // existing title already matches or was edited.
8436                                land::set_pr_text_verbatim(
8437                                    &winner.worktree,
8438                                    &url,
8439                                    &pr.title,
8440                                    &pr.body,
8441                                )
8442                                .await
8443                                .map(|()| url)
8444                            } else {
8445                                if title != pr.title
8446                                    && (!is_review_run(&self.state)
8447                                        || leaked
8448                                            .as_deref()
8449                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8450                                    && let Err(e) =
8451                                        if self.state.github_text.as_ref().is_some_and(|g| {
8452                                            g.resolved
8453                                                && g.title
8454                                                && g.choice == Some(GithubTextChoice::Original)
8455                                        }) {
8456                                            // The owner chose to post this title unmodified.
8457                                            land::set_pr_title_verbatim(
8458                                                &winner.worktree,
8459                                                &url,
8460                                                &pr.title,
8461                                            )
8462                                            .await
8463                                        } else {
8464                                            land::set_pr_title(
8465                                                &mut self.state,
8466                                                &winner.worktree,
8467                                                &url,
8468                                                &pr.title,
8469                                            )
8470                                            .await
8471                                        }
8472                                {
8473                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8474                                    self.state
8475                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8476                                }
8477                                Ok(url)
8478                            }
8479                        }
8480                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8481                    };
8482                    match out {
8483                        Ok(url) => MergeOutcome {
8484                            mode,
8485                            ok: true,
8486                            detail: url,
8487                            empty: false,
8488                        },
8489                        Err(e) => MergeOutcome {
8490                            mode,
8491                            ok: false,
8492                            detail: e.to_string(),
8493                            empty: false,
8494                        },
8495                    }
8496                }
8497            }
8498        };
8499
8500        self.state.status = match (mode, outcome.ok) {
8501            (MergeMode::None, _) => RunStatus::Ready,
8502            (_, true) if outcome.empty => RunStatus::Ready,
8503            (MergeMode::Pr, true) => RunStatus::Ready,
8504            (_, true) => RunStatus::Merged,
8505            (_, false) => RunStatus::Blocked,
8506        };
8507        self.state.event(
8508            "merge",
8509            format!(
8510                "{:?}: {}",
8511                mode,
8512                outcome.detail.lines().next().unwrap_or("")
8513            ),
8514        );
8515        self.state.merge = Some(outcome);
8516        self.state.save()?;
8517
8518        // The PR is open and the run would historically stop here, leaving the
8519        // operator to watch checks, feed review comments back to a fixer, and
8520        // merge. That was done by hand six times in one session before this
8521        // existed. Opt-in, because merging is the one irreversible thing magi
8522        // can do to a repository.
8523        if self.state.config.graph.land
8524            && mode == MergeMode::Pr
8525            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8526        {
8527            self.run_land().await?;
8528        }
8529        // `run_land` may have left `status` at `Landing` - still waiting on
8530        // CI or the owner's approval, not actually settled - so this has to
8531        // read whatever `status` ended up as here, not the `Merged` this
8532        // function set a few lines up.
8533        self.settle_questions();
8534        Ok(())
8535    }
8536
8537    /// Enter `land`.
8538    ///
8539    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8540    /// a resumed run's re-entry. `land::land` itself is what serialises the
8541    /// two git-mutating moments inside the loop — the rebase push and
8542    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8543    /// to hold a lock across the whole call, and doing so would serialise
8544    /// this run's CI wait against a *different* run's land-approval resume
8545    /// in the same repository, which is exactly the "must not wait on
8546    /// another task" property the daemon's slot-freeing exists to give.
8547    async fn run_land(&mut self) -> Result<()> {
8548        crate::pr_head::ensure_owner(&self.state)?;
8549        let url = self
8550            .state
8551            .pr
8552            .as_ref()
8553            .map(|pr| pr.url.clone())
8554            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8555            .unwrap_or_default();
8556        let url = url.lines().next().unwrap_or("").trim().to_owned();
8557        if !url.starts_with("http") {
8558            return Ok(());
8559        }
8560        // A land failure is not a lost run: the work is on a branch and the
8561        // pull request is open, which is exactly where a human takes over.
8562        loop {
8563            match land::land(&mut self.state, &url).await {
8564                Ok(_)
8565                    if self
8566                        .state
8567                        .merge_repairs
8568                        .last()
8569                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8570                {
8571                    if self.repair_merge_review().await? {
8572                        continue;
8573                    }
8574                }
8575                Ok(pr) if self.state.parked => {
8576                    // `land` already saved the parked marker; nothing here
8577                    // overrides `status` back to a terminal value while an
8578                    // approval is still outstanding.
8579                    let _ = pr;
8580                }
8581                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8582                    self.state.status = RunStatus::Superseded;
8583                    self.state.save()?;
8584                }
8585                Ok(pr) => {
8586                    self.state.status = match pr.state {
8587                        land::PrLifecycle::Merged => RunStatus::Merged,
8588                        _ => RunStatus::Blocked,
8589                    };
8590                    // Downstream of a confirmed merge only - see
8591                    // `bump::should_release_bump`'s own doc for why this one
8592                    // check covers all three of `land`'s success paths.
8593                    // Best-effort: the run already landed, so a failure here
8594                    // (the decision call, `gh`, `cargo`) is recorded and never
8595                    // turns a landed run into a failed one.
8596                    if bump::should_release_bump(self.state.status)
8597                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8598                    {
8599                        // The event is the run's own record. Not-eligible cases
8600                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8601                        // `Err` is a bump that was tried and failed:
8602                        // `after_merge` itself raises the operator notice for
8603                        // that, whether or not a release PR exists yet.
8604                        self.state
8605                            .event("bump", format!("release bump skipped: {e:#}"));
8606                    }
8607                    // Independent of the bump, and best-effort in the same way:
8608                    // findings the merge left open become follow-up tasks.
8609                    if self.state.status == RunStatus::Merged {
8610                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8611                    }
8612                    self.state.save()?;
8613                }
8614                Err(e) => {
8615                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8616                        return Err(e);
8617                    }
8618                    self.state.status = RunStatus::Blocked;
8619                    self.state.event("land", format!("gave up: {e}"));
8620                    self.state.save()?;
8621                }
8622            }
8623            break;
8624        }
8625        Ok(())
8626    }
8627
8628    /// One repair cycle stays under the existing driver/task claim. An
8629    /// interrupted fixer is rolled back and held rather than run twice.
8630    async fn repair_merge_review(&mut self) -> Result<bool> {
8631        use crate::run::MergeRepairPhase as Phase;
8632        crate::pr_head::ensure_owner(&self.state)?;
8633        let i = self.state.merge_repairs.len() - 1;
8634        let request = self.state.merge_repairs[i].clone();
8635        if request.phase == Phase::Fixing {
8636            let restored = match self.state.winner().cloned() {
8637                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8638                None => Err(anyhow::anyhow!("repair has no winning branch")),
8639            };
8640            let why = match restored {
8641                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8642                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8643                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8644            };
8645            self.hold_merge_repair(i, why)?;
8646            return Ok(false);
8647        }
8648        let failure = if i >= self.state.config.graph.land_rounds
8649            || self.state.config.graph.review_rounds == 0
8650        {
8651            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8652        } else {
8653            None
8654        };
8655        if let Some(why) = failure {
8656            self.hold_merge_repair(i, why)?;
8657            return Ok(false);
8658        }
8659        if request.phase == Phase::Requested {
8660            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8661                Ok(pr) => pr,
8662                Err(e) => {
8663                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8664                    return Ok(false);
8665                }
8666            };
8667            self.state.merge_repairs[i].phase = Phase::Fixing;
8668            self.state.land_approval = None;
8669            self.state.land_armed_head = None;
8670            self.state.save()?;
8671            // Serialize the exact records, including finding ids/details,
8672            // reconsideration reasons, final votes and verification output.
8673            let evidence = serde_json::to_string_pretty(&request)?;
8674            let reason = format!(
8675                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8676            );
8677            let budget = self.state.config.graph.land_rounds;
8678            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8679            match result {
8680                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8681                Ok(land::Fixed::Failed(why)) => {
8682                    self.hold_merge_repair(
8683                        i,
8684                        format!(
8685                            "repair failed: {why}; inspect the branch and run an explicit review"
8686                        ),
8687                    )?;
8688                    return Ok(false);
8689                }
8690                Err(e) => {
8691                    self.hold_merge_repair(
8692                        i,
8693                        format!(
8694                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8695                        ),
8696                    )?;
8697                    return Ok(false);
8698                }
8699            }
8700            // Even no change gets a real new reviewer wave. The previous
8701            // evidence remains in the request; a former clean/concluded round
8702            // cannot short-circuit the new review or gate.
8703            self.begin_merge_repair_review(i)?;
8704        }
8705        self.review_loop().await?;
8706        self.gate().await?;
8707        if self.resource_halt()? || self.state.parked {
8708            return Ok(false);
8709        }
8710        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8711            != Some(RunStatus::Gating)
8712            || !self.state.gate_allows_progress()
8713        {
8714            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8715            return Ok(false);
8716        }
8717        // Review and gate may have added commits after the initial repair.
8718        // Publish exactly their final head to the existing PR, with a lease.
8719        let winner = self
8720            .state
8721            .winner()
8722            .cloned()
8723            .context("repair has no winning branch")?;
8724        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8725        if self.state.gate_head.as_deref() != Some(after.as_str())
8726            || !git::is_clean(&winner.worktree).await?
8727        {
8728            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8729            return Ok(false);
8730        }
8731        let expected = self
8732            .state
8733            .pr_pushed_head
8734            .as_deref()
8735            .unwrap_or(&request.head)
8736            .to_owned();
8737        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8738        // Push the checked commit itself: a local ref moving after the check
8739        // must not change which commit the forge receives.
8740        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8741        let refspec = format!("{after}:refs/heads/{public}");
8742        let push = git::git_raw(
8743            &winner.worktree,
8744            &["push", &lease, &self.state.config.merge.remote, &refspec],
8745        )
8746        .await;
8747        match push {
8748            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8749            Ok(out) => {
8750                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8751                return Ok(false);
8752            }
8753            Err(e) => {
8754                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8755                return Ok(false);
8756            }
8757        }
8758        self.state.merge_repairs[i].phase = Phase::Complete;
8759        self.state.status = RunStatus::Landing;
8760        self.state.event(
8761            "repair",
8762            "new review and gate completed; fresh owner merge decision required",
8763        );
8764        self.state.save()?;
8765        Ok(true)
8766    }
8767
8768    fn merge_repair_artifacts(&self) -> PathBuf {
8769        let artifacts = agent::artifacts_dir(&self.state.dir());
8770        match self
8771            .state
8772            .merge_repairs
8773            .last()
8774            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8775        {
8776            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8777            None => artifacts,
8778        }
8779    }
8780
8781    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8782        self.state.reviews.clear();
8783        self.state.gate.clear();
8784        self.state.gate_ran = false;
8785        self.state.gate_head = None;
8786        self.state.gate_fixes.clear();
8787        self.state.gate_repair_e2e = None;
8788        self.state.pre_gate.clear();
8789        self.state.pre_gate_commit = None;
8790        self.state.contested_handoff = None;
8791        self.state.land_reruns = None;
8792        self.state.land_approval = None;
8793        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8794        self.state.status = RunStatus::Reviewing;
8795        self.state.save()
8796    }
8797
8798    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8799        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8800        self.state.merge_repairs[index].failure = Some(why.clone());
8801        self.state.status = RunStatus::Blocked;
8802        self.state.event("repair", why);
8803        self.state.save()
8804    }
8805
8806    // -------------------------------------------------------------- helpers
8807
8808    /// Fetch or create a seat, keeping its conversation across nodes.
8809    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8810        if let Some(existing) = self.state.seats.get(key)
8811            && existing.agent == agent
8812        {
8813            return existing.clone();
8814        }
8815        // A seat that changes agent mints its session id from the agent too,
8816        // like a handover: the old agent's uuid is already taken by the CLI.
8817        let fresh = if self.state.seats.contains_key(key) {
8818            handover_seat(key, agent, self.state.next_seat_seed())
8819        } else {
8820            SeatState::new(key, agent, self.state.seed)
8821        };
8822        self.state.seats.insert(key.to_owned(), fresh.clone());
8823        fresh
8824    }
8825
8826    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8827    /// seat to another roster agent, in which case that agent. Nodes that
8828    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8829    /// reconsideration) must keep talking to whoever answered it, not slip
8830    /// back to the agent that failed it.
8831    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8832        match self.state.seats.get(key) {
8833            Some(s) if s.agent != spec.id => {
8834                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8835            }
8836            _ => spec,
8837        }
8838    }
8839
8840    /// A candidate rendered for judging, with the leak policy applied.
8841    fn view(&self, c: &Candidate) -> CandidateView {
8842        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8843            .unwrap_or_default();
8844        let (patch, _) = blind::sanitize_patch(
8845            &format!("candidate {} patch", c.label),
8846            &raw,
8847            &self.state.config.blind,
8848        );
8849        CandidateView {
8850            label: c.label,
8851            branch: c.branch.clone(),
8852            summary: c.summary.clone(),
8853            stat: c.stat.clone(),
8854            patch,
8855        }
8856    }
8857
8858    /// The full candidate set as prompt text, for seats with no live session.
8859    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8860        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8861        prompt::judge(
8862            "(see above)",
8863            &views,
8864            self.roles.judges.len(),
8865            base_short,
8866            "en",
8867        )
8868    }
8869
8870    /// The final-vote prompt with everything a seat that has no session of its
8871    /// own needs: the candidates, the seat's own ranking and reasons, and the
8872    /// anonymised deliberation it took part in (only when there was one, so a
8873    /// handed-over seat never sees more than the seat it replaces did). The
8874    /// `Final vote` heading stays first.
8875    fn vote_prompt_full(
8876        &self,
8877        j: usize,
8878        labels: &[char],
8879        language: &str,
8880        candidates: &[Candidate],
8881        base_short: &str,
8882    ) -> String {
8883        let mut text = format!(
8884            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8885            prompt::final_vote(labels, language),
8886            self.state.instruction,
8887            self.candidate_block(candidates, base_short)
8888        );
8889        if let Some(own) = self
8890            .state
8891            .judgements
8892            .get(j)
8893            .filter(|r| !r.ranking.is_empty())
8894        {
8895            let reasons = own
8896                .reasons
8897                .iter()
8898                .map(|(k, v)| format!("- {k}: {v}"))
8899                .collect::<Vec<_>>()
8900                .join("\n");
8901            text.push_str(&format!(
8902                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8903                own.ranking.iter().collect::<String>(),
8904                if reasons.is_empty() {
8905                    ""
8906                } else {
8907                    ", because:\n"
8908                }
8909            ));
8910        }
8911        if !self.state.deliberation.is_empty() {
8912            text.push_str("\n# What was argued before this vote\n");
8913            for t in self.transcript(&[], j) {
8914                text.push_str(&format!(
8915                    "\n## {}{}\n\n{}\n",
8916                    t.who,
8917                    if t.is_self { " (you)" } else { "" },
8918                    t.body.trim()
8919                ));
8920            }
8921        }
8922        text
8923    }
8924
8925    /// Anonymised transcript for judge `self_idx`.
8926    ///
8927    /// The initial rankings are always the opening statements. Seeding them
8928    /// only when no turn had been taken yet meant every judge after the first
8929    /// argued against a single voice instead of against the actual split — the
8930    /// disagreement is the information, so it is always on the table.
8931    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8932        let mut turns = Vec::new();
8933        for j in &self.state.judgements {
8934            if j.ranking.is_empty() {
8935                continue;
8936            }
8937            let reasons = j
8938                .reasons
8939                .iter()
8940                .map(|(k, v)| format!("- {k}: {v}"))
8941                .collect::<Vec<_>>()
8942                .join("\n");
8943            turns.push(Turn {
8944                who: format!("Judge {} (opening ranking)", j.judge),
8945                is_self: j.judge == self_idx + 1,
8946                body: format!(
8947                    "Ranked {}{}{reasons}",
8948                    j.ranking.iter().collect::<String>(),
8949                    if reasons.is_empty() {
8950                        ""
8951                    } else {
8952                        ", because:\n"
8953                    }
8954                ),
8955            });
8956        }
8957        for t in self
8958            .state
8959            .deliberation
8960            .iter()
8961            .flat_map(|r| r.turns.iter())
8962            .chain(current)
8963        {
8964            turns.push(Turn {
8965                who: format!("Judge {}", t.judge),
8966                is_self: t.judge == self_idx + 1,
8967                body: t.body.clone(),
8968            });
8969        }
8970        turns
8971    }
8972}
8973
8974/// Does this seat still hold the context a follow-up prompt would rely on?
8975fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8976    agent::has_session(spec.kind, seat, sessions)
8977}
8978
8979/// The next entry in `roster` after `start`, never wrapping back to the
8980/// front, whose id is not in `tried` yet.
8981///
8982/// Starts one past `start` rather than at the front of `roster`: `start` is
8983/// the seat's own original position, and a seat whose candidate slot already
8984/// sits on the roster's second entry must fall through to the third next, not
8985/// restart at the first — which is very likely a different candidate's own
8986/// agent already. Never wraps back past `start`, for the same reason: an
8987/// entry earlier in the roster than the seat's own position is almost
8988/// certainly some *other* candidate slot's own agent, and once the tail of
8989/// the roster is exhausted there are no more untried agents for *this* seat
8990/// to fall through to — the caller's fallback chain ends there, exactly as
8991/// "no further untried agents remain in the list for that seat" asks for.
8992///
8993/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8994/// the same id twice (an operator's `roles.implementers` typo, or a
8995/// `[[agents]]` list reused across roles) must not let
8996/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8997/// over `roster` either finds an untried id or runs out, so this always
8998/// terminates regardless of duplicates.
8999fn next_untried_in_roster<'a>(
9000    roster: &'a [AgentSpec],
9001    start: usize,
9002    tried: &BTreeSet<String>,
9003) -> Option<&'a AgentSpec> {
9004    roster
9005        .get(start + 1..)?
9006        .iter()
9007        .find(|s| !tried.contains(&s.id))
9008}
9009
9010/// The successor for a seat, shared by all four seat kinds (implement, judge,
9011/// review, advise). `others` holds the ids that *currently* occupy the other
9012/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9013/// sees them), so roster entries beyond the seat count act as spares: a failed
9014/// seat goes to an agent no other seat holds whenever the roster permits.
9015///
9016/// `carried` is `Some` only for the review loop's carried failure history.
9017/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9018/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9019/// whole roster: untried and not another seat's occupant; (3) the plain walk
9020/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9021/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9022/// a better outcome than an empty seat, and it keeps the answer to "is there
9023/// a successor at all" exactly what it was before occupants were considered,
9024/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9025/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9026/// carried failure is a cheaper bet than doubling an agent on the panel, and
9027/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9028/// the same round are handled one at a time, so a seat later in the batch
9029/// sees an earlier one's new occupant but not yet its own freed agent.
9030fn pick_successor<'a>(
9031    roster: &'a [AgentSpec],
9032    start: usize,
9033    tried: &BTreeSet<String>,
9034    carried: Option<&BTreeSet<String>>,
9035    others: &BTreeSet<String>,
9036) -> Option<&'a AgentSpec> {
9037    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9038    roster
9039        .get(start + 1..)
9040        .and_then(|tail| {
9041            tail.iter()
9042                .filter(free)
9043                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9044        })
9045        .or_else(|| {
9046            carried
9047                .is_some()
9048                .then(|| roster.iter().find(free))
9049                .flatten()
9050        })
9051        .or_else(|| match carried {
9052            Some(c) => next_for_seat(roster, start, tried, c),
9053            None => next_untried_in_roster(roster, start, tried),
9054        })
9055}
9056
9057/// The next agent for a seat that carries its failure history across rounds
9058/// (the review loop). `round_tried` is this round's own bound and starts
9059/// empty every round; `carried_failed` only decides priority.
9060///
9061/// First: an id walking forward from `start`, never wrapping, that is neither
9062/// tried this round nor failed in an earlier one. Only when that is exhausted
9063/// does it rescue: the first roster id (in roster order, so this one *does*
9064/// look before `start`) not yet tried this round, which is by then a carried
9065/// failure. Each id is rescued at most once per round, so it cannot loop.
9066fn next_for_seat<'a>(
9067    roster: &'a [AgentSpec],
9068    start: usize,
9069    round_tried: &BTreeSet<String>,
9070    carried_failed: &BTreeSet<String>,
9071) -> Option<&'a AgentSpec> {
9072    roster
9073        .get(start + 1..)?
9074        .iter()
9075        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9076        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9077}
9078
9079/// Where a reviewer seat starts a round: the agent that last answered it when
9080/// it is still on the roster and not marked failed, else the spec's own agent
9081/// unless it failed, else the next roster agent that has not failed, else the
9082/// spec's own agent again (the whole roster failed). Ids no longer on the
9083/// roster are ignored. An empty roster has no handover, so the spec stands.
9084fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9085    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9086        return spec;
9087    };
9088    let ok = |id: &str| !h.failed.contains(id);
9089    if let Some(last) = h.last_ok.as_deref()
9090        && ok(last)
9091        && let Some(s) = roster.iter().find(|s| s.id == last)
9092    {
9093        return s.clone();
9094    }
9095    if ok(&spec.id) {
9096        return spec;
9097    }
9098    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9099    roster
9100        .get(from + 1..)
9101        .into_iter()
9102        .flatten()
9103        .chain(roster.iter())
9104        .find(|s| ok(&s.id))
9105        .cloned()
9106        .unwrap_or(spec)
9107}
9108
9109/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9110/// seed so a CLI that mints its session id up front (`--session-id`) never
9111/// reuses the uuid the previous agent already opened under the same seat key.
9112pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9113    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9114}
9115
9116/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9117/// so the classifier below and the code that builds the message cannot drift.
9118const TIMED_OUT: &str = "timed out";
9119
9120impl FailClass {
9121    /// `None` for an answer; otherwise how the turn failed.
9122    fn of(out: &AgentOutcome) -> Option<Self> {
9123        match out {
9124            AgentOutcome::Ok(_) => None,
9125            AgentOutcome::Quota(_) => Some(Self::Quota),
9126            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9127            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9128            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9129            // Not a failure of the agent. Callers check for it first; `None`
9130            // here means "no class to hand over on".
9131            AgentOutcome::ResourceBlocked(_) => None,
9132        }
9133    }
9134
9135    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9136    fn stem_word(&self) -> &'static str {
9137        match self {
9138            Self::Quota => "quota",
9139            _ => "handover",
9140        }
9141    }
9142}
9143
9144/// The message's first line with its variable parts removed — digit runs and
9145/// path-like tokens — so "exit code 2" and "exit code 7" read
9146/// as one kind of failure.
9147fn failure_signature(msg: &str) -> String {
9148    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9149    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9150    // the historical machine key while the process diagnosis uses readable prose.
9151    if let Some(code) = line
9152        .strip_prefix("process exited with no usable output (exit code ")
9153        .and_then(|s| s.strip_suffix(')'))
9154        .and_then(|s| s.parse::<i32>().ok())
9155    {
9156        line = format!("exited with some({code}) and no usable output");
9157    } else if matches!(
9158        line.as_str(),
9159        "process exited with no usable output (no exit code reported)"
9160            | "process exited with no usable output (exit code unavailable)"
9161    ) {
9162        line = "exited with none and no usable output".to_owned();
9163    }
9164    let mut out = Vec::new();
9165    for word in line.split_whitespace() {
9166        if word.contains('/') || word.contains('\\') {
9167            out.push("<path>".to_owned());
9168            continue;
9169        }
9170        let mut w = String::new();
9171        let mut in_digits = false;
9172        for c in word.chars() {
9173            if c.is_ascii_digit() {
9174                if !in_digits {
9175                    w.push('#');
9176                }
9177                in_digits = true;
9178            } else {
9179                in_digits = false;
9180                w.push(c);
9181            }
9182        }
9183        out.push(w);
9184    }
9185    out.join(" ").chars().take(120).collect()
9186}
9187
9188/// Whether a seat that just failed with `cur` may go to the next roster agent.
9189/// A quota or a timeout always may. Any other failure may not when the agent
9190/// before it failed the same way: an error the prompt causes would otherwise
9191/// walk the whole roster. `prev` is the class of the immediately preceding
9192/// agent's failure, so a quota or timeout in between breaks the run of
9193/// identical failures by itself.
9194fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9195    match cur {
9196        FailClass::Quota | FailClass::Timeout => true,
9197        FailClass::Other(_) => prev != Some(cur),
9198    }
9199}
9200
9201/// A short human reason for a failed outcome, for the handover record.
9202fn fail_reason(out: &AgentOutcome) -> String {
9203    match out {
9204        AgentOutcome::Ok(_) => String::new(),
9205        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9206        AgentOutcome::Dropped(o) => format!(
9207            "the CLI dropped the stream ({})",
9208            o.dropped
9209                .as_ref()
9210                .map(|d| d.why.as_str())
9211                .unwrap_or("it ended without delivering its answer")
9212        ),
9213        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9214            e.lines().next().unwrap_or("").chars().take(160).collect()
9215        }
9216    }
9217}
9218
9219/// Note one handover in the run: the structured record and, in the timeline,
9220/// the sentence a person reads. A quota keeps the wording it always had.
9221pub(crate) fn record_handover(
9222    state: &mut RunState,
9223    node: &str,
9224    seat: &str,
9225    from: &str,
9226    to: &str,
9227    class: &FailClass,
9228    reason: &str,
9229) {
9230    let message = if *class == FailClass::Quota {
9231        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9232    } else {
9233        format!("{seat}: handed over {from} -> {to} ({reason})")
9234    };
9235    state.event(node, message);
9236    state.handovers.push(Handover {
9237        at: Timestamp::now(),
9238        node: node.to_owned(),
9239        seat: seat.to_owned(),
9240        from: from.to_owned(),
9241        to: to.to_owned(),
9242        reason: reason.to_owned(),
9243    });
9244}
9245
9246/// Did this reply report running a command whose own CLI never confirmed an
9247/// exit status?
9248///
9249/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9250/// command *finished* (see that type's own doc), so this can only be `true`
9251/// for a command whose completion event carried no readable exit code — not
9252/// for one that simply is not mentioned at all. That is the one signal this
9253/// crate can read, from the same record `state.jobs` renders, about a reply
9254/// standing next to work its own CLI cannot vouch for finishing; it is
9255/// deliberately not a check on the exit code's *value* (a fixer legitimately
9256/// runs a command that fails mid-iteration before it succeeds) and not a
9257/// guess at a command still running in the background (which emits no event
9258/// at all, and so leaves no evidence here to find).
9259fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9260    commands.iter().any(|c| c.exit_code.is_none())
9261}
9262
9263/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9264/// trusted as a verified no-op — the adoption guard's own text-level half.
9265///
9266/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9267/// timed out): a marker only earns the benefit of the doubt from a turn the
9268/// CLI itself vouches for finishing properly, the same house style
9269/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9270/// *fix* report to for `commands`. A candidate that timed out, exited
9271/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9272/// is, whatever prose it wrote — this returns `None` before it ever looks at
9273/// `text`. The remaining guards (the tree really is empty, the evidence is
9274/// non-empty) are the caller's: this only reads what the reply *claimed*.
9275fn verified_noop_claim(
9276    usable: bool,
9277    commands: &[agent::CommandEvidence],
9278    text: &str,
9279) -> Option<String> {
9280    (usable && !has_unconfirmed_command(commands))
9281        .then(|| verdict::verified_noop(text))
9282        .flatten()
9283}
9284
9285fn short(commit: &str) -> String {
9286    commit.chars().take(7).collect()
9287}
9288
9289fn make_executable(path: &Path) -> Result<()> {
9290    #[cfg(unix)]
9291    {
9292        use std::os::unix::fs::PermissionsExt as _;
9293        let mut perms = std::fs::metadata(path)?.permissions();
9294        perms.set_mode(0o755);
9295        std::fs::set_permissions(path, perms)?;
9296    }
9297    #[cfg(not(unix))]
9298    {
9299        let _ = path;
9300    }
9301    Ok(())
9302}
9303
9304/// What every seat in one batch shares: where the answers are attributed, the
9305/// prompt overlay they inherit, and the build cache they are told to use.
9306///
9307/// A struct rather than four more parameters: `wave` also needs the run's
9308/// state (to record who is answering right now) and the attempt number, and
9309/// eight positional arguments is both unreadable and a clippy error.
9310struct WaveCtx<'a> {
9311    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9312    /// paid for it.
9313    run: &'a str,
9314    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9315    node: &'a str,
9316    prompts: &'a Prompts,
9317    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9318    cache: Option<&'a Path>,
9319    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9320    /// `JobRecord::round`. `None` for every other node.
9321    round: Option<usize>,
9322    /// Carry each seat's failed-agent history across waves (the review loop
9323    /// only): start-of-round priority and handover choice read
9324    /// [`RunState::seat_history`], and every answer or failure writes it.
9325    carry_seats: bool,
9326    /// The run's park / interrupt requests, so a wait for a shared resource
9327    /// can be abandoned the moment the operator asks to park.
9328    pauses: Option<(&'a Pause, &'a Pause)>,
9329}
9330
9331/// Shared scheduling policy for operations and spawned seat invocations.
9332fn repository_wait(
9333    state: &RunState,
9334    pauses: Option<(&Pause, &Pause)>,
9335) -> crate::repo_lock::WaitPolicy {
9336    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9337    let inherited = crate::repo_lock::WaitPolicy::current();
9338    crate::repo_lock::WaitPolicy::new(
9339        Duration::from_secs(state.config.graph.timeout_resource_wait),
9340        move || {
9341            inherited.cancelled()
9342                || pauses
9343                    .as_ref()
9344                    .is_some_and(|(a, b)| a.parked() || b.parked())
9345        },
9346    )
9347}
9348
9349fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9350    if error.is::<crate::repo_lock::ResourceBlocked>() {
9351        state.resource_block = Some(crate::run::ResourceBlock {
9352            cause: crate::run::ResourceBlockCause::Other,
9353            node: "repository".into(),
9354            kind: if error.to_string().contains("disk pressure") {
9355                "disk pressure"
9356            } else {
9357                "repository coordination"
9358            }
9359            .into(),
9360            detail: error.to_string(),
9361            at: Timestamp::now(),
9362        });
9363        state.event("resource", error.to_string());
9364    }
9365}
9366
9367async fn coordinate_operation(
9368    state: &mut RunState,
9369    cwd: &Path,
9370    wait: &crate::repo_lock::WaitPolicy,
9371) -> Result<Option<crate::repo_lock::SeatLease>> {
9372    let result =
9373        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9374            .await;
9375    if let Err(e) = &result {
9376        record_repository_block(state, e);
9377    }
9378    result
9379}
9380
9381/// Run one job, honouring the parallelism budget.
9382async fn run_one(
9383    job: SeatJob,
9384    sem: Arc<Semaphore>,
9385    ctx: &WaveCtx<'_>,
9386    state: &mut RunState,
9387    attempt: usize,
9388) -> (SeatState, AgentOutcome) {
9389    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9390        .await
9391        .pop()
9392        .expect("one job in, one result out");
9393    (seat, out)
9394}
9395
9396/// Run every job concurrently, capped by the semaphore, preserving order.
9397///
9398/// Every seat in the batch is recorded into [`RunState::active`] before the
9399/// wave starts and cleared as each answer lands, so the run's own record says
9400/// who is still being waited on rather than only who finished.
9401async fn wave(
9402    jobs: Vec<SeatJob>,
9403    sem: Arc<Semaphore>,
9404    ctx: &WaveCtx<'_>,
9405    state: &mut RunState,
9406    attempt: usize,
9407) -> Vec<(usize, SeatState, AgentOutcome)> {
9408    let WaveCtx {
9409        run,
9410        node,
9411        prompts,
9412        cache,
9413        round,
9414        carry_seats: _,
9415        pauses,
9416    } = *ctx;
9417    for job in &jobs {
9418        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9419    }
9420    if let Err(e) = state.save() {
9421        // A failed persist of "who is answering right now" must not abort the
9422        // wave: the seats are already being asked, and the alternative is
9423        // losing the answers to save a status line nobody may even be
9424        // watching.
9425        tracing::warn!("could not persist in-progress seats: {e:#}");
9426    }
9427    // Hold the shared build cache's lease for the whole batch, not per job:
9428    // several candidates (an implement wave) or a fixer legitimately share
9429    // one cache concurrently within this run, and that stays untouched — a
9430    // single lease taken once for the whole wave and released once it is
9431    // done is what stops a *different* borrower (another run's own wave, its
9432    // e2e/gate, a human's `magi review`) from interleaving a build into the
9433    // same directory while this one is in flight. Best-effort, not
9434    // all-or-nothing: a wave that cannot get the lease within its own
9435    // longest job's budget still runs — an hour of paid implementer calls is
9436    // not thrown away over cache contention — but every write-allowed seat
9437    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9438    // below), the same fallback a read-only seat always gets, rather than
9439    // building into a directory this run was never granted. The identity
9440    // record is still invalidated below either way, so the next tracked
9441    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9442    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9443    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9444    // never borrowed from a job's agent budget (and never charged to it either,
9445    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9446    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9447    let cache_guard = if let Some(cache_dir) = cache {
9448        if jobs_had_a_writer {
9449            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9450            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9451            match acquire_cache_lease(
9452                state,
9453                cache_dir,
9454                &owner,
9455                resource_limit,
9456                node,
9457                node,
9458                &cancel,
9459            )
9460            .await
9461            {
9462                Ok(Some(g)) => Some(g),
9463                // Parking: leave without spawning anything. The run parks at
9464                // the next boundary and the seats are asked afresh on resume.
9465                Ok(None) => {
9466                    if let Some(block) = &mut state.resource_block {
9467                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9468                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9469                        };
9470                    }
9471                    return block_wave(
9472                        jobs,
9473                        state,
9474                        node,
9475                        attempt,
9476                        "parked while waiting for the shared build cache",
9477                    );
9478                }
9479                // An unreadable lease is not contention: keep the old
9480                // fallback (no shared cache for this wave's seats).
9481                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9482                Err(e) => {
9483                    // Writing seats may not build into a cache this run does
9484                    // not hold, and an agent that cannot be given one is not
9485                    // asked at all: the run stops resumably instead.
9486                    let why = format!("{e:#}");
9487                    state.event(
9488                        "resource",
9489                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9490                    );
9491                    return block_wave(jobs, state, node, attempt, &why);
9492                }
9493            }
9494        } else {
9495            None
9496        }
9497    } else {
9498        None
9499    };
9500    // Every job keeps its whole budget: the wait above was not the agent's.
9501    // Restart the seats' clocks so "started" means "being spawned now".
9502    state.seats_spawning(node, attempt);
9503    if let Err(e) = state.save() {
9504        tracing::warn!("could not persist the start of a wave: {e:#}");
9505    }
9506    // Queueing behind this wave's own seats for a parallelism slot is normal
9507    // (each of them is bounded by its own agent timeout); only a wait beyond
9508    // all of them plus the resource limit means the slots never came back.
9509    let slot_limit = resource_limit
9510        + jobs
9511            .iter()
9512            .map(|j| j.timeout)
9513            .max()
9514            .unwrap_or_default()
9515            .saturating_mul(jobs.len() as u32);
9516    let mut set = tokio::task::JoinSet::new();
9517    // Seats announce themselves here the moment they hold a parallelism
9518    // permit, i.e. just before their CLI is spawned.
9519    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9520    let overlay = prompts.overlay(node);
9521    let repository_wait = repository_wait(state, pauses);
9522    for (i, mut job) in jobs.into_iter().enumerate() {
9523        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9524        if cache.is_some() {
9525            job.prompt.push('\n');
9526            job.prompt
9527                .push_str(&prompt::build_cache_note(node, job.allow_write));
9528        }
9529        let sem = Arc::clone(&sem);
9530        let run = run.to_owned();
9531        let node = node.to_owned();
9532        // Only implementers were told about the task's attachments, so only
9533        // their seats get the directory widened for reading.
9534        let attachments = if node == "implement" {
9535            state.attachments.clone()
9536        } else {
9537            Vec::new()
9538        };
9539        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9540        // `prompt::build_cache_note`'s doc for why setting it anyway is
9541        // exactly how a sandboxed reviewer's write refusal got reported as a
9542        // defect in the patch, not a property of its own seat. And a
9543        // write-allowed one is handed it only when the lease above was
9544        // actually acquired: a wave that could not get it (`cache_guard` is
9545        // `None`, see its own comment) must not send seats to build into a
9546        // directory this run does not hold - that is the exact concurrent,
9547        // unmanaged-write race this module exists to prevent, not something
9548        // "proceeding anyway" is allowed to reintroduce.
9549        let cache = cache
9550            .filter(|_| job.allow_write && cache_guard.is_some())
9551            .map(Path::to_path_buf);
9552        let spawned_tx = spawned_tx.clone();
9553        let repository_wait = repository_wait.clone();
9554        let coordination = state.config.coordination.clone();
9555        let fetched_base = state.fetched_base.clone();
9556        set.spawn(async move {
9557            // A wait for a parallelism slot is scheduling, not the agent: it
9558            // is bounded by its own limit and ends as a resource block, never
9559            // as a timeout of the agent that was not started.
9560            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9561                Ok(p) => p,
9562                Err(_) => {
9563                    return (
9564                        i,
9565                        job.seat,
9566                        AgentOutcome::ResourceBlocked(format!(
9567                            "no parallel slot became free within {}s",
9568                            slot_limit.as_secs()
9569                        )),
9570                        false,
9571                    );
9572                }
9573            };
9574            spawned_tx.send(job.seat.key.clone()).ok();
9575            let mut seat = job.seat;
9576            let out = repository_wait
9577                .scope(agent::invoke(
9578                    &job.spec,
9579                    &mut seat,
9580                    &Invocation {
9581                        coordination: coordination.as_ref(),
9582                        fetched_base: fetched_base.as_ref(),
9583                        cwd: &job.cwd,
9584                        prompt: &job.prompt,
9585                        timeout: job.timeout,
9586                        allow_write: job.allow_write,
9587                        sessions: job.sessions,
9588                        artifacts: &job.artifacts,
9589                        stem: &job.stem,
9590                        run: &run,
9591                        node: &node,
9592                        cache_dir: cache.as_deref(),
9593                        attachments: &attachments,
9594                        writable: &[],
9595                    },
9596                ))
9597                .await;
9598            let interrupted = out
9599                .as_ref()
9600                .err()
9601                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9602            let out = match out {
9603                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9604                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9605                // Billed work the CLI failed to hand over is not an ordinary
9606                // failure, but its text is the CLI's raw error JSON, not an
9607                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9608                // read it as one by forgetting to check. `usable()` is always
9609                // false here (dropped implies an empty response), so this has
9610                // to be checked before the catch-all `Failed` below or the
9611                // one shape this exists for is lost with the rest.
9612                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9613                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9614                Ok(o) => AgentOutcome::Failed(format!(
9615                    "process exited with no usable output ({})",
9616                    crate::notices::exit_status(o.exit_code)
9617                )),
9618                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9619                    AgentOutcome::ResourceBlocked(e.to_string())
9620                }
9621                Err(e) => AgentOutcome::Failed(e.to_string()),
9622            };
9623            (i, seat, out, interrupted)
9624        });
9625    }
9626    drop(spawned_tx);
9627    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9628    loop {
9629        let joined = tokio::select! {
9630            Some(key) = spawned_rx.recv() => {
9631                // The clock of this seat starts now, not when it queued.
9632                state.seat_spawned(&key);
9633                if let Err(e) = state.save() {
9634                    tracing::warn!("could not persist a seat's start: {e:#}");
9635                }
9636                continue;
9637            }
9638            joined = set.join_next() => joined,
9639        };
9640        let Some(joined) = joined else {
9641            break;
9642        };
9643        let (i, seat, out, interrupted) = match joined {
9644            Ok(v) => v,
9645            // No seat to clear: a panicked task never reported which one it
9646            // was. The defensive sweep below this loop is what stops that
9647            // seat's `active` entry from surviving forever.
9648            Err(e) => {
9649                tracing::error!("agent task panicked: {e}");
9650                continue;
9651            }
9652        };
9653        state.seat_finished(&seat.key);
9654        if interrupted
9655            && node == "implement"
9656            && let Some(label) = state
9657                .candidates
9658                .iter()
9659                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9660                .map(|candidate| candidate.label)
9661        {
9662            state.candidate_preservation.insert(
9663                label,
9664                "interrupted implementation awaiting persistence".into(),
9665            );
9666            state.interrupted_implementations.insert(label);
9667        }
9668        if let AgentOutcome::ResourceBlocked(why) = &out
9669            && state.resource_block.is_none()
9670        {
9671            state.resource_block = Some(crate::run::ResourceBlock {
9672                cause: crate::run::ResourceBlockCause::Other,
9673                node: node.to_owned(),
9674                kind: if why.contains("disk pressure") {
9675                    "disk pressure"
9676                } else if why.contains("repository coordination") {
9677                    "repository coordination"
9678                } else {
9679                    "parallel slot"
9680                }
9681                .to_owned(),
9682                detail: why.clone(),
9683                at: Timestamp::now(),
9684            });
9685            let activity = if interrupted {
9686                "agent execution interrupted"
9687            } else {
9688                "no agent was started"
9689            };
9690            state.event("resource", format!("{node}: {activity} - {why}"));
9691        }
9692        record_jobs(state, node, round, &seat.key, &out);
9693        if let Err(e) = state.save() {
9694            tracing::warn!("could not persist a seat's completion: {e:#}");
9695        }
9696        if collected.len() <= i {
9697            collected.resize_with(i + 1, || None);
9698        }
9699        collected[i] = Some((i, seat, out));
9700    }
9701    // Belt-and-braces for the panic branch above: every seat this exact batch
9702    // started shares this `(node, attempt)` pair, and every seat that finished
9703    // normally already cleared itself, so anything left tagged with it here
9704    // can only be a panicked task's leftover. Cleared unconditionally rather
9705    // than left to read as still answering forever.
9706    if state
9707        .active
9708        .values()
9709        .any(|a| a.node == node && a.attempt == attempt)
9710    {
9711        state
9712            .active
9713            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9714        if let Err(e) = state.save() {
9715            tracing::warn!("could not persist the end of a wave: {e:#}");
9716        }
9717    }
9718    // Whether or not the lease above was actually held, several worktrees
9719    // may just have built into the cache with nothing here able to name one
9720    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9721    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9722    // clean it might not have strictly needed; trusting a stale match would
9723    // cost it a wrong answer.
9724    if let Some(cache_dir) = cache
9725        && jobs_had_a_writer
9726    {
9727        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9728    }
9729    if let Some(guard) = cache_guard {
9730        guard.release();
9731    }
9732    collected.into_iter().flatten().collect()
9733}
9734
9735/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9736/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9737/// this is data collection, not the fix-specific completion contract in
9738/// [`Runner::continue_fix_report`], and applies regardless of which node
9739/// asked.
9740///
9741/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9742/// evidence from; `Failed` does not, and correctly contributes nothing — a
9743/// timeout or crash is not itself evidence about a command the seat may have
9744/// started.
9745fn record_jobs(
9746    state: &mut RunState,
9747    node: &str,
9748    round: Option<usize>,
9749    seat: &str,
9750    out: &AgentOutcome,
9751) {
9752    let commands: &[agent::CommandEvidence] = match out {
9753        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9754        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9755    };
9756    let checked_at = Timestamp::now();
9757    for c in commands {
9758        state.jobs.push(JobRecord {
9759            node: node.to_owned(),
9760            round,
9761            seat: seat.to_owned(),
9762            id: c.id.clone(),
9763            description: c.description.clone(),
9764            checked_at,
9765            status: match c.exit_code {
9766                Some(0) => JobStatus::Completed,
9767                Some(_) => JobStatus::Failed,
9768                None => JobStatus::Unknown,
9769            },
9770            exit_code: c.exit_code,
9771            result_summary: c.result_summary.clone(),
9772            source: c.source.clone(),
9773        });
9774    }
9775}
9776
9777/// Is a review round clean, given how many reviewer seats answered against
9778/// how many the round expected?
9779///
9780/// A seat that never answered (timeout, crash, unparsable output) is not a
9781/// seat that read the patch and found nothing — treating it as such is
9782/// exactly the bug this function exists to close. Under the default `block`
9783/// policy a missing seat can never be clean; `warn` still requires the seats
9784/// that *did* answer to have found nothing blocking and verification to be
9785/// green.
9786///
9787/// `quota_missing` narrows that `block` default for exactly one cause of
9788/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9789/// a session limit lifts by the very next round buys nothing — the seat is
9790/// asked again with the same quota — so once every missing seat is accounted
9791/// for by a quota loss (and at least one seat *did* answer, so a decision has
9792/// something to rest on) the round is decided on the panel that could answer,
9793/// same as `warn` would. A panel that lost every seat to quota is not
9794/// decided here: `answered == 0` falls through to the existing `block`
9795/// fallback so a fully collapsed panel still waits rather than landing on no
9796/// review at all.
9797fn round_is_clean(
9798    blocking: usize,
9799    e2e_ok: bool,
9800    answered: usize,
9801    expected: usize,
9802    quota_missing: usize,
9803    policy: IncompleteReviewPolicy,
9804) -> bool {
9805    if blocking != 0 || !e2e_ok {
9806        return false;
9807    }
9808    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9809        return true;
9810    }
9811    answered > 0 && expected - answered <= quota_missing
9812}
9813
9814/// The review loop's own conclusion, derived entirely from its persisted
9815/// round records and the round budget that produced them — never from
9816/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9817/// recomputes the identical answer regardless of what an earlier node in the
9818/// same walk, or a previous walk, did to `status`.
9819///
9820/// `None` while more rounds remain to try, including when review never ran
9821/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9822/// gone clean, or the budget is spent, or the tree has stopped moving (see
9823/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9824///
9825/// - An incomplete panel that raised nothing is missing input, not a
9826///   verified tree — never a hand-off candidate, whatever verification said
9827///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9828/// - Otherwise, green e2e on the last round hands off (see
9829///   [`Runner::stop_reviewing`]); red e2e blocks.
9830///
9831/// A last round whose own verification is still `ResourceBlocked` — magi
9832/// itself never got a command to run, not evidence the patch is broken —
9833/// is neither: this returns `None` for it too, the same as "more rounds
9834/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9835/// handling of that shape) instead of this cheap recomputation guessing a
9836/// verdict a real attempt never produced.
9837pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9838    if max_rounds == 0
9839        || reviews.last().is_some_and(|r| {
9840            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9841        })
9842    {
9843        return Some(RunStatus::Gating);
9844    }
9845    let last = reviews.last()?;
9846    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9847    if reviews.len() < max_rounds && !stagnant {
9848        return None;
9849    }
9850    if last.incomplete() && last.blocking == 0 {
9851        return Some(RunStatus::Blocked);
9852    }
9853    if last.e2e_status() == E2eStatus::ResourceBlocked
9854        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9855    {
9856        return None;
9857    }
9858    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9859        RunStatus::Gating
9860    } else {
9861        RunStatus::Blocked
9862    })
9863}
9864
9865/// How long a re-ask may take, given the budget the first attempt had.
9866///
9867/// A `nudged` retry is a request to restate an answer the seat has already
9868/// worked out: it carries no new work, so it does not deserve the original
9869/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9870/// 133 seconds while a third sat for over ten minutes on a resumed session
9871/// holding 410 KB of prior output - and because the retry had inherited the
9872/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9873/// judging round whose other seats were long finished.
9874///
9875/// A quarter of the budget, with a floor so that a deliberately short timeout
9876/// does not collapse to nothing. A retry that re-sends the whole prompt
9877/// (because the seat kept no context) is the original job again, and keeps the
9878/// original budget.
9879pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9880    if nudged {
9881        (full / 4).max(Duration::from_secs(120)).min(full)
9882    } else {
9883        full
9884    }
9885}
9886
9887/// Run a wave and parse each reply, re-asking the seats whose reply was
9888/// unusable.
9889///
9890/// The re-ask is a nudge rather than the whole prompt again when the seat still
9891/// holds its conversation, which is the difference between a cheap retry and
9892/// paying for the entire candidate set twice.
9893///
9894/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9895/// successor in `roster` is handed to it instead of being re-asked: the
9896/// handover is the retry. A seat with no successor left (a single-agent
9897/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9898/// the asks to one seat in one node number at most
9899/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9900/// is asked once (a dropped stream is nudged first), and only the last agent
9901/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9902/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9903/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9904/// other failure stops the chain when the previous agent failed the same way
9905/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9906/// [`has_context`] is false and the job's own full prompt and full budget are
9907/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9908/// intermediate ones are not losses) and is returned as a failure like any
9909/// other absent seat — the caller decides whether the panel still has a
9910/// quorum. An empty `roster` disables handover: failures are nudged as they
9911/// always were, and a quota is simply lost. A reply that fails to parse or
9912/// validate is the prompt's doing and is only ever nudged, never handed over.
9913///
9914/// The returned [`SeatState`] names the agent that answered (or tried last).
9915#[allow(clippy::too_many_arguments)]
9916async fn ask_json_wave<T>(
9917    jobs: Vec<SeatJob>,
9918    sem: Arc<Semaphore>,
9919    retries: usize,
9920    roster: &[AgentSpec],
9921    ctx: &WaveCtx<'_>,
9922    losses: &mut Vec<QuotaLoss>,
9923    state: &mut RunState,
9924    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9925) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9926where
9927    T: serde::de::DeserializeOwned + Send + 'static,
9928{
9929    ask_wave_with(
9930        jobs,
9931        sem,
9932        retries,
9933        roster,
9934        ctx,
9935        losses,
9936        state,
9937        &|text: &str| {
9938            let v = verdict::extract_json::<T>(text)?;
9939            validate(&v)?;
9940            Ok(v)
9941        },
9942    )
9943    .await
9944}
9945
9946/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9947/// a node whose answer is prose (deliberation) shares the same handover,
9948/// failure classification, quota bookkeeping and bounds instead of a copy.
9949///
9950/// A seat handed to another roster agent is sent the job's `handover` prompt
9951/// (when it has one) rather than `prompt`: the new agent has no session, so a
9952/// resume-style prompt would be incomplete. That holds for the handover ask
9953/// and for every nudge to that agent whose `has_context` is false.
9954#[allow(clippy::too_many_arguments)]
9955async fn ask_wave_with<T>(
9956    jobs: Vec<SeatJob>,
9957    sem: Arc<Semaphore>,
9958    retries: usize,
9959    roster: &[AgentSpec],
9960    ctx: &WaveCtx<'_>,
9961    losses: &mut Vec<QuotaLoss>,
9962    state: &mut RunState,
9963    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9964) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9965where
9966    T: Send + 'static,
9967{
9968    let n = jobs.len();
9969    let originals: Vec<SeatJob> = jobs;
9970    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9971    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9972    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9973    // answer, N once it has gone through N nudges. Read back once this
9974    // returns, so a caller building a history record (`ReviewRecord`) can
9975    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9976    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9977    // field's own doc.
9978    let mut nudges: Vec<usize> = vec![0; n];
9979    // The agent now occupying each seat, the ids it has already been through,
9980    // where in the roster the walk began, the class of the last failure, and
9981    // the stem word of a handover not yet asked (full prompt, full budget).
9982    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9983    let mut tried: Vec<BTreeSet<String>> = specs
9984        .iter()
9985        .map(|s| BTreeSet::from([s.id.clone()]))
9986        .collect();
9987    let starts: Vec<usize> = specs
9988        .iter()
9989        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9990        .collect();
9991    let carry = ctx.carry_seats && !roster.is_empty();
9992    // Carried across rounds: ids that failed the seat earlier, and how the
9993    // last failure went (so a repeat of it is not handed over again).
9994    let carried: Vec<BTreeSet<String>> = originals
9995        .iter()
9996        .map(|j| {
9997            state
9998                .seat_history
9999                .get(&j.seat.key)
10000                .filter(|_| carry)
10001                .map(|h| h.failed.clone())
10002                .unwrap_or_default()
10003        })
10004        .collect();
10005    let mut prev: Vec<Option<FailClass>> = originals
10006        .iter()
10007        .map(|j| {
10008            state
10009                .seat_history
10010                .get(&j.seat.key)
10011                .filter(|_| carry)
10012                .and_then(|h| h.last_fail.clone())
10013        })
10014        .collect();
10015    let next_agent =
10016        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10017            let others: BTreeSet<String> = specs
10018                .iter()
10019                .enumerate()
10020                .filter(|(j, _)| *j != i)
10021                .map(|(_, s)| s.id.clone())
10022                .collect();
10023            pick_successor(
10024                roster,
10025                starts[i],
10026                tried,
10027                carry.then(|| &carried[i]),
10028                &others,
10029            )
10030            .cloned()
10031        };
10032    let mut fresh: Vec<Option<String>> = vec![None; n];
10033    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10034    let mut pending: Vec<usize> = (0..n).collect();
10035
10036    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10037    // asks: an agent with a successor is asked once and handed over, and only
10038    // a seat with no successor spends `retries` nudges on the same agent. This
10039    // only guarantees the loop's own termination whatever those say.
10040    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10041    for round in 0..max_rounds {
10042        if pending.is_empty() {
10043            break;
10044        }
10045        let mut batch = Vec::with_capacity(pending.len());
10046        let mut renudged: Vec<&str> = Vec::new();
10047        for &i in &pending {
10048            let src = &originals[i];
10049            // A seat now held by another agent than the job named has no
10050            // session of its own: it gets the full-context prompt whenever
10051            // it is asked in full (the handover ask, a nudge it cannot
10052            // resume).
10053            let full: &str = match &src.handover {
10054                Some(h) if specs[i].id != src.spec.id => h,
10055                _ => &src.prompt,
10056            };
10057            // The prompt and the budget are one decision: a nudge restates
10058            // finished work, a re-sent prompt redoes it.
10059            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10060                (
10061                    full.to_owned(),
10062                    src.timeout,
10063                    format!("{}-{word}-{}", src.stem, specs[i].id),
10064                )
10065            } else if nudges[i] == 0 {
10066                (full.to_owned(), src.timeout, src.stem.clone())
10067            } else {
10068                renudged.push(src.seat.key.as_str());
10069                let why = done[i]
10070                    .as_ref()
10071                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10072                    .unwrap_or_else(|| "no parsable answer".to_owned());
10073                let nudge = prompt::nudge(&why);
10074                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10075                let prompt = if nudged {
10076                    nudge
10077                } else {
10078                    format!("{full}\n\n---\n\n{nudge}")
10079                };
10080                (
10081                    prompt,
10082                    retry_budget(src.timeout, nudged),
10083                    format!("{}-retry{}", src.stem, nudges[i]),
10084                )
10085            };
10086            batch.push(SeatJob {
10087                spec: specs[i].clone(),
10088                seat: seats[i].clone(),
10089                cwd: src.cwd.clone(),
10090                prompt,
10091                timeout,
10092                allow_write: src.allow_write,
10093                sessions: src.sessions,
10094                artifacts: src.artifacts.clone(),
10095                stem,
10096                handover: None,
10097            });
10098        }
10099
10100        if !renudged.is_empty() {
10101            state.event(
10102                ctx.node,
10103                format!("retry {round}: re-asking {}", renudged.join(", ")),
10104            );
10105        }
10106        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10107        let mut still = Vec::new();
10108        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10109            seats[i] = seat;
10110            // No agent was asked: nothing to hand over, nudge, remember or
10111            // charge. The caller sees `resource_block` and stops the node.
10112            if let AgentOutcome::ResourceBlocked(why) = &out {
10113                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10114                continue;
10115            }
10116            let class = FailClass::of(&out);
10117            // A dropped stream is nudged first (the conversation is still
10118            // there to pick up); only a seat whose nudges are spent hands over.
10119            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10120                && nudges[i] < retries
10121                && !roster.is_empty();
10122            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10123                let next = should_hand_over(prev[i].as_ref(), &cur)
10124                    .then(|| next_agent(i, &tried[i], &specs))
10125                    .flatten();
10126                if carry {
10127                    let h = state
10128                        .seat_history
10129                        .entry(originals[i].seat.key.clone())
10130                        .or_default();
10131                    h.failed.insert(specs[i].id.clone());
10132                    h.last_fail = Some(cur.clone());
10133                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10134                        h.last_ok = None;
10135                    }
10136                    // Saved before the next agent is asked, so a restart in
10137                    // between does not forget who failed.
10138                    if let Err(e) = state.save() {
10139                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10140                    }
10141                }
10142                if let Some(next) = next {
10143                    record_handover(
10144                        state,
10145                        ctx.node,
10146                        &originals[i].seat.key,
10147                        &specs[i].id,
10148                        &next.id,
10149                        &cur,
10150                        &fail_reason(&out),
10151                    );
10152                    tried[i].insert(next.id.clone());
10153                    prev[i] = Some(cur.clone());
10154                    seats[i] =
10155                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10156                    specs[i] = next;
10157                    fresh[i] = Some(cur.stem_word().to_owned());
10158                    nudges[i] = 0;
10159                    done[i] = Some(Err(anyhow::anyhow!(
10160                        "handed over after: {}",
10161                        fail_reason(&out)
10162                    )));
10163                    still.push(i);
10164                    continue;
10165                }
10166            }
10167            if carry
10168                && !nudge_first
10169                && let Some(cur) = class.clone()
10170            {
10171                // The chain ended here (no successor, or a repeated failure).
10172                let h = state
10173                    .seat_history
10174                    .entry(originals[i].seat.key.clone())
10175                    .or_default();
10176                h.failed.insert(specs[i].id.clone());
10177                h.last_fail = Some(cur);
10178            }
10179            let parsed = match out {
10180                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10181                AgentOutcome::Quota(o) => {
10182                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10183                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10184                }
10185                // Not a parseable answer: the nudge loop re-asks it, which is
10186                // exactly what a dropped stream needs. Just don't hand its raw
10187                // error JSON to `extract_json`.
10188                AgentOutcome::Dropped(o) => {
10189                    let why = o
10190                        .dropped
10191                        .as_ref()
10192                        .map(|d| d.why.as_str())
10193                        .unwrap_or("the CLI ended the stream without delivering its answer");
10194                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10195                }
10196                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10197                    Err(anyhow::anyhow!(e))
10198                }
10199            };
10200            let quota = class == Some(FailClass::Quota);
10201            let failed = parsed.is_err();
10202            done[i] = Some(parsed);
10203            if carry && !failed {
10204                let h = state
10205                    .seat_history
10206                    .entry(originals[i].seat.key.clone())
10207                    .or_default();
10208                h.failed.remove(&specs[i].id);
10209                h.last_ok = Some(specs[i].id.clone());
10210                h.last_fail = None;
10211            }
10212            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10213            // fail the same way; and never re-ask a seat that already parsed.
10214            // A failed agent that still has a successor is not re-asked
10215            // either: the handover was its remedy and has just been refused
10216            // (the chain stops on a repeated failure class). A seat with no
10217            // successor left (a single-agent roster, the roster's tail, or an
10218            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10219            let agent_failure = class.is_some()
10220                && !nudge_first
10221                && !roster.is_empty()
10222                && next_agent(i, &tried[i], &specs).is_some();
10223            if failed && !quota && !agent_failure && nudges[i] < retries {
10224                nudges[i] += 1;
10225                still.push(i);
10226            }
10227        }
10228        pending = still;
10229    }
10230
10231    // One loss per seat whose chain ended on a quota: the intermediate ones
10232    // were absorbed by a handover and are not losses.
10233    for (i, q) in last_quota.into_iter().enumerate() {
10234        if let Some(reset) = q {
10235            losses.push(QuotaLoss {
10236                seat: originals[i].seat.key.clone(),
10237                node: ctx.node.to_owned(),
10238                at: Timestamp::now(),
10239                reset,
10240            });
10241        }
10242    }
10243
10244    seats
10245        .into_iter()
10246        .zip(done)
10247        .zip(nudges)
10248        .map(|((seat, res), attempts)| {
10249            (
10250                seat,
10251                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10252                attempts,
10253            )
10254        })
10255        .collect()
10256}
10257
10258/// Did the wave end without asking any agent (see [`block_wave`])?
10259fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10260    results
10261        .iter()
10262        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10263}
10264
10265/// Settle a wave that never spawned anything: every job answers
10266/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10267/// else about the seat (session, history, quota) is touched.
10268fn block_wave(
10269    jobs: Vec<SeatJob>,
10270    state: &mut RunState,
10271    node: &str,
10272    attempt: usize,
10273    why: &str,
10274) -> Vec<(usize, SeatState, AgentOutcome)> {
10275    state.resource_wait = None;
10276    state
10277        .active
10278        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10279    if let Err(e) = state.save() {
10280        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10281    }
10282    jobs.into_iter()
10283        .enumerate()
10284        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10285        .collect()
10286}
10287
10288/// Acquire the shared build cache's lease, waiting out contention within
10289/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10290/// unbounded wait is never acceptable).
10291///
10292/// A first, non-blocking check happens before ever waiting; if it finds the
10293/// lease busy, that fact is logged as a `verify` event *and* flushed with
10294/// [`RunState::save`] immediately — not only once the wait finally succeeds
10295/// or gives up — so a `magi show` run by a different process while this one
10296/// is still waiting reads a `run.json` that says so, rather than whatever it
10297/// looked like before the wait started. The same applies to the terminal
10298/// failure: logged and saved before this returns `Err`, so a caller that
10299/// could not get the lease at all still leaves a legible record of why.
10300async fn acquire_cache_lease(
10301    state: &mut RunState,
10302    cache_dir: &Path,
10303    owner: &crate::cache::Owner,
10304    budget: Duration,
10305    node: &str,
10306    context: &str,
10307    cancel: &(dyn Fn() -> bool + Sync),
10308) -> Result<Option<crate::cache::Guard>> {
10309    let home = crate::run::home();
10310    let started = Instant::now();
10311    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10312        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10313        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10314        Err(e) => {
10315            state.event(
10316                "verify",
10317                format!("{context}: could not check the shared build cache: {e:#}"),
10318            );
10319            if let Err(e2) = state.save() {
10320                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10321            }
10322            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10323        }
10324    };
10325    state.event(
10326        "verify",
10327        format!(
10328            "{context}: waiting for the shared build cache at {} ({})",
10329            cache_dir.display(),
10330            busy.describe()
10331        ),
10332    );
10333    // Seats of this node are waiting, not running: say so, so the countdown a
10334    // reader sees only ever measures an agent that has been spawned.
10335    state.resource_wait = Some(crate::run::ResourceWait {
10336        node: context.to_owned(),
10337        kind: "shared build cache".to_owned(),
10338        holder: busy.describe(),
10339        since: Timestamp::now(),
10340        limit_secs: budget.as_secs(),
10341    });
10342    if let Err(e) = state.save() {
10343        tracing::warn!("could not persist a cache wait: {e:#}");
10344    }
10345    let remaining = budget.saturating_sub(started.elapsed());
10346    let waited = crate::cache::wait_for_unless(
10347        &home,
10348        cache_dir,
10349        owner,
10350        remaining,
10351        Duration::from_secs(5),
10352        cancel,
10353    )
10354    .await;
10355    state.resource_wait = None;
10356    let cause = match &waited {
10357        Ok(Some(_)) => None,
10358        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10359        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10360            Some(crate::run::ResourceBlockCause::Timeout {
10361                waited_secs: started.elapsed().as_secs(),
10362                limit_secs: budget.as_secs(),
10363            })
10364        }
10365        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10366    };
10367    if let Some(cause) = cause {
10368        let detail = match &waited {
10369            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10370            Err(error) => format!("{error:#}"),
10371        };
10372        state.resource_block = Some(crate::run::ResourceBlock {
10373            cause,
10374            node: node.to_owned(),
10375            kind: "shared build cache".into(),
10376            detail,
10377            at: Timestamp::now(),
10378        });
10379        state.event(
10380            "resource",
10381            state.resource_block.as_ref().unwrap().description(),
10382        );
10383    }
10384    if let Err(error) = state.save() {
10385        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10386    }
10387    waited
10388}
10389
10390/// The lease state could not be read at all (unreadable lease file, a
10391/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10392/// known to be contended, so a wave degrades to running without a shared cache
10393/// instead of stopping on a hold that [`crate::triage`] could never release.
10394#[derive(Debug)]
10395struct CacheCheckError(String);
10396
10397impl std::fmt::Display for CacheCheckError {
10398    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10399        f.write_str(&self.0)
10400    }
10401}
10402
10403impl std::error::Error for CacheCheckError {}
10404
10405/// Keep a specific acquisition outcome when the caller adds command evidence.
10406fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10407    state
10408        .resource_block
10409        .get_or_insert_with(|| crate::run::ResourceBlock {
10410            cause: crate::run::ResourceBlockCause::Other,
10411            node: node.to_owned(),
10412            kind: "shared build cache".into(),
10413            detail: format!("{error:#}"),
10414            at: Timestamp::now(),
10415        });
10416    if let Err(error) = state.save() {
10417        tracing::warn!("could not persist cache resource block: {error:#}");
10418    }
10419}
10420
10421/// Run `body` — a verify command batch — while holding the shared build
10422/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10423/// never interleave with another borrower's build against the same
10424/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10425/// timeout, or a human's own `magi review`. See the `cache` module doc for
10426/// why this matters more than Cargo's own per-target locking covers — two
10427/// *different* worktrees building the same package name/version into one
10428/// cache directory is a staleness bug, not a lock contention one.
10429///
10430/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10431/// agent waves do. Repository scheduling also has its own bounded,
10432/// cancellable resource-wait policy. Contention must not consume the
10433/// verification budget: `body` receives the full `budget` after both leases
10434/// are acquired and the cache is fresh. Without a shared cache, only
10435/// repository coordination is acquired before handing over the full budget.
10436///
10437/// A lease that cannot be acquired within the resource wait is reported as a single
10438/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10439/// verification — the same shape a spawn failure already takes in
10440/// [`run_commands`], so a caller need not special-case it.
10441#[allow(clippy::too_many_arguments)]
10442async fn with_cache_lease<'s, F, Fut>(
10443    state: &'s mut RunState,
10444    cache_dir: Option<&Path>,
10445    node: &str,
10446    seat: &str,
10447    worktree: &Path,
10448    head: &str,
10449    budget: Duration,
10450    context: &str,
10451    wait: &crate::repo_lock::WaitPolicy,
10452    body: F,
10453) -> Result<(Vec<CommandOutcome>, bool)>
10454where
10455    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10456    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10457{
10458    // Match editing waves: cache first, repository second. In particular,
10459    // pre_gate must never hold the repository while waiting for a wave's cache.
10460    let started = Instant::now();
10461    let home = crate::run::home();
10462    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10463    let guard = if let Some(cache_dir) = cache_dir {
10464        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10465        match acquire_cache_lease(
10466            state,
10467            cache_dir,
10468            &owner,
10469            resource_limit,
10470            node,
10471            context,
10472            &|| wait.cancelled(),
10473        )
10474        .await
10475        .and_then(|g| {
10476            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10477        }) {
10478            Ok(g) => Some(g),
10479            Err(e) => {
10480                record_cache_error(state, node, &e);
10481                return Ok((
10482                    vec![CommandOutcome {
10483                        command: "(waiting for the shared build cache)".into(),
10484                        code: None,
10485                        output_tail: e.to_string(),
10486                        duration_ms: started.elapsed().as_millis() as u64,
10487                        resource_blocked: true,
10488                    }],
10489                    false,
10490                ));
10491            }
10492        }
10493    } else {
10494        None
10495    };
10496    let _coordination = coordinate_operation(state, worktree, wait).await?;
10497    if let Some(cache_dir) = cache_dir {
10498        let identity = crate::cache::Identity::new(worktree, head);
10499        if let Err(e) = crate::cache::ensure_fresh(
10500            &home,
10501            cache_dir,
10502            &identity,
10503            &state.config.verify.cargo_manifests(),
10504        ) {
10505            record_cache_error(state, node, &e);
10506            state.event(
10507                "verify",
10508                format!(
10509                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10510                    worktree.display(),
10511                    short(head)
10512                ),
10513            );
10514            if let Some(guard) = guard {
10515                guard.release();
10516            }
10517            return Ok((
10518                vec![CommandOutcome {
10519                    command: "(confirming the shared build cache is fresh)".into(),
10520                    code: None,
10521                    output_tail: e.to_string(),
10522                    duration_ms: started.elapsed().as_millis() as u64,
10523                    resource_blocked: true,
10524                }],
10525                false,
10526            ));
10527        }
10528    }
10529    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10530    if !timed_out_pids.is_empty() {
10531        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10532    }
10533    // Drop repository coordination before releasing the cache, reversing the
10534    // acquisition order. A cancelled/error future also drops both guards.
10535    drop(_coordination);
10536    if let Some(guard) = guard {
10537        guard.release();
10538    }
10539    Ok((outcomes, retried))
10540}
10541
10542/// Poll `pids` — commands [`run_commands`] reports as still running when its
10543/// own timeout elapsed — until every one is confirmed gone, or
10544/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10545///
10546/// Real confirmation where confirmation is possible, not a substitute for
10547/// full process-tree observation: a grandchild the timed-out process spawned
10548/// and that survives independently of it is invisible to a pid check the
10549/// same way it always was, and continuing to observe and collect *that*
10550/// stays a different piece of work with its own owner. This only narrows a
10551/// fixed blind wait into an actual check of the pids this process does know
10552/// about.
10553async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10554    wait_for_pids_with(
10555        pids,
10556        crate::proc::pid_alive,
10557        LEASE_RELEASE_POLL,
10558        LEASE_RELEASE_MAX_WAIT,
10559    )
10560    .await;
10561}
10562
10563/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10564/// interval and ceiling supplied by the caller, so the polling *logic* -
10565/// returns as soon as every pid reports dead, gives up at the ceiling
10566/// otherwise - is testable on millisecond durations without asking the real
10567/// OS about a pid at all.
10568async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10569    pids: &[u32],
10570    alive: F,
10571    poll: Duration,
10572    max_wait: Duration,
10573) {
10574    let deadline = Instant::now() + max_wait;
10575    loop {
10576        if pids.iter().all(|&pid| !alive(pid)) {
10577            return;
10578        }
10579        if Instant::now() >= deadline {
10580            return;
10581        }
10582        tokio::time::sleep(poll).await;
10583    }
10584}
10585
10586/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10587/// admission that it could not even get a verify command to run, as opposed
10588/// to evidence the command actually produced? A caller that would otherwise
10589/// read a resource-blocked outcome as a red command must check this first:
10590/// see [`Runner::gate`], which retries rather than records `Blocked` when
10591/// this is true.
10592fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10593    outcomes.iter().any(|o| o.resource_blocked)
10594}
10595
10596/// What [`Runner::gate_fix_round`] decided.
10597pub(crate) enum GateFix {
10598    /// The tree changed and `verify.e2e` is still green: run the gate again.
10599    Retry,
10600    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10601    /// last failure stands and the run ends blocked.
10602    Stop,
10603    /// `verify.e2e` could not run after the fix (magi's own contention):
10604    /// decide nothing now, a later reentry retries.
10605    Defer,
10606}
10607
10608/// Is every red command in `outcomes` an ordinary failure the code could
10609/// explain: it ran, exited non-zero, and said something?
10610///
10611/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10612/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10613/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10614/// what the command is stays the gate's business.
10615fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10616    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10617    red.peek().is_some()
10618        && red.all(|o| {
10619            !o.resource_blocked
10620                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10621                && !o.output_tail.trim().is_empty()
10622        })
10623}
10624
10625/// Describe one verify command's outcome for the event log, distinguishing a
10626/// build/link failure — the toolchain never produced a binary to run — from
10627/// an actual test failure, since only the latter is a verdict on the patch.
10628fn e2e_outcome_label(o: &CommandOutcome) -> String {
10629    if o.ok() {
10630        return "pass".to_owned();
10631    }
10632    let reason = if o.build_failed() {
10633        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10634    } else {
10635        format!("FAIL ({:?})", o.code)
10636    };
10637    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10638}
10639
10640/// Run `verify.e2e`, retrying once if the first attempt could not build or
10641/// link — a build/link failure is frequently a race against a shared
10642/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10643/// `verify` event per command, tagged with `context` (normally `"round N"`)
10644/// so the two call sites that need this — the ordinary per-round leg in
10645/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10646/// it will ever call a round green — read identically in the event log.
10647async fn run_e2e_with_retry(
10648    state: &mut RunState,
10649    shell: &[String],
10650    commands: &[String],
10651    worktree: &Path,
10652    timeout: Duration,
10653    context: &str,
10654) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10655    let (mut e2e, mut timed_out_pids) = run_commands(
10656        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10657    )
10658    .await;
10659    for o in &e2e {
10660        state.event(
10661            "verify",
10662            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10663        );
10664    }
10665    // A build/link failure is not a verdict on the patch — it is frequently a
10666    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10667    // one retry before letting a red like that decide the round.
10668    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10669    if verify_retried {
10670        state.event(
10671            "verify",
10672            format!(
10673                "{context}: verify could not build/link, not a test result — retrying once \
10674                 before concluding"
10675            ),
10676        );
10677        let retried = run_commands(
10678            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10679        )
10680        .await;
10681        e2e = retried.0;
10682        // Both attempts' timeouts matter, not just the last one: the first
10683        // attempt's descendants may still be alive alongside the retry's.
10684        timed_out_pids.extend(retried.1);
10685        for o in &e2e {
10686            state.event(
10687                "verify",
10688                format!(
10689                    "{context}: retry `{}` -> {}",
10690                    o.command,
10691                    e2e_outcome_label(o)
10692                ),
10693            );
10694        }
10695    }
10696    (e2e, verify_retried, timed_out_pids)
10697}
10698
10699/// Run configured shell commands in `cwd`, in order. The second element is
10700/// the pid of every command that hit `timeout` and was still running when
10701/// this stopped waiting on it (best-effort: `None` when the platform did not
10702/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10703/// that releases a shared resource afterward needs to know.
10704///
10705/// Records `task` into [`RunState::active`] at every command boundary
10706/// (`RunState::task_command`) and clears it once the whole list has run
10707/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10708/// for minutes with no seat and no output of its own to show for it (see
10709/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10710/// apart from "not yet run" without this), and this is the only place that
10711/// knows which command is running right now and how many are left. Three
10712/// saves per command — start, not per second — matching the same "only at a
10713/// boundary" rule [`wave`] already follows for seats.
10714#[allow(clippy::too_many_arguments)]
10715async fn run_commands(
10716    state: &mut RunState,
10717    node: &str,
10718    task: &str,
10719    attempt: usize,
10720    shell: &[String],
10721    commands: &[String],
10722    cwd: &Path,
10723    timeout: Duration,
10724) -> (Vec<CommandOutcome>, Vec<u32>) {
10725    if commands.is_empty() {
10726        // Nothing to mark as running and nothing to clear — an empty list
10727        // means "not configured", and touching `active` (or the disk) over
10728        // that would be a write for every round of a repo with no
10729        // `verify.e2e` / `verify.gate` commands at all.
10730        return (Vec::new(), Vec::new());
10731    }
10732    let mut out = Vec::new();
10733    let mut timed_out_pids = Vec::new();
10734    let total = commands.len();
10735    for (idx, command) in commands.iter().enumerate() {
10736        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10737        if let Err(e) = state.save() {
10738            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10739        }
10740        let started = Instant::now();
10741        let mut cmd = tokio::process::Command::new(&shell[0]);
10742        cmd.quiet();
10743        // Verify the checkout as a standalone caller. Worker ownership and
10744        // pinned executable selection belong to this driver, not to tests or
10745        // tools it runs. Preserve home/seat context and Running's containment.
10746        for key in crate::worker::INVOCATION_ENV {
10747            cmd.env_remove(key);
10748        }
10749        cmd.args(&shell[1..])
10750            .arg(command)
10751            .current_dir(cwd)
10752            .stdin(std::process::Stdio::null())
10753            .stdout(std::process::Stdio::piped())
10754            .stderr(std::process::Stdio::piped())
10755            .kill_on_drop(true);
10756        let mut outputs = Vec::new();
10757        let protected = (|| -> Result<()> {
10758            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10759                outputs.push(guard);
10760            }
10761            if let Some(cache) = state.config.cache_dir() {
10762                outputs.push(crate::retention::OutputLease::cache(
10763                    &crate::run::home(),
10764                    &cache,
10765                    false,
10766                )?);
10767            }
10768            Ok(())
10769        })();
10770        if let Err(e) = protected {
10771            out.push(CommandOutcome {
10772                command: command.clone(),
10773                code: None,
10774                output_tail: format!("output ownership unavailable: {e:#}"),
10775                duration_ms: 0,
10776                resource_blocked: true,
10777            });
10778            continue;
10779        }
10780        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10781        let (code, body) = match spawned {
10782            Ok(mut running) => {
10783                let pid = running.child().id();
10784                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10785                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10786                let mut paths = vec![cwd.to_owned()];
10787                if let Some(cache) = state.config.cache_dir() {
10788                    paths.push(cache);
10789                }
10790                let mut pressure = None;
10791                let status = tokio::select! {
10792                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10793                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10794                        pressure = Some(reason); None
10795                    }
10796                };
10797                let (code, failure) = match status {
10798                    Some(Ok(Ok(status))) => (
10799                        status.code(),
10800                        status
10801                            .code()
10802                            .is_none()
10803                            .then(|| format!("command terminated: {status}")),
10804                    ),
10805                    Some(Ok(Err(error))) => {
10806                        (None, Some(format!("failed waiting for command: {error}")))
10807                    }
10808                    Some(Err(_)) => {
10809                        // Leased cleanup confirms process-tree death itself.
10810                        // Unleased commands still need the caller's PID check
10811                        // after kill-on-drop, before releasing cache ownership.
10812                        if !running.coordinated()
10813                            && let Some(pid) = pid
10814                        {
10815                            timed_out_pids.push(pid);
10816                        }
10817                        (
10818                            None,
10819                            Some(format!("timed out after {}s", timeout.as_secs())),
10820                        )
10821                    }
10822                    None => (None, None),
10823                };
10824                running.finish().await;
10825                if let Some(reason) = pressure {
10826                    state.resource_block = Some(crate::run::ResourceBlock {
10827                        cause: crate::run::ResourceBlockCause::Other,
10828                        node: node.to_owned(),
10829                        kind: "disk pressure".to_owned(),
10830                        detail: reason.clone(),
10831                        at: Timestamp::now(),
10832                    });
10833                    out.push(CommandOutcome {
10834                        command: command.clone(),
10835                        code: None,
10836                        output_tail: reason,
10837                        duration_ms: started.elapsed().as_millis() as u64,
10838                        resource_blocked: true,
10839                    });
10840                    break;
10841                }
10842                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10843                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10844                if let Some(failure) = failure {
10845                    body.push('\n');
10846                    body.push_str(&failure);
10847                }
10848                (code, body)
10849            }
10850            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10851        };
10852        out.push(CommandOutcome {
10853            command: command.clone(),
10854            code,
10855            output_tail: tail(&body, OUTPUT_TAIL),
10856            duration_ms: started.elapsed().as_millis() as u64,
10857            resource_blocked: false,
10858        });
10859    }
10860    state.task_finished(task);
10861    if let Err(e) = state.save() {
10862        tracing::warn!("could not persist the end of {task}: {e:#}");
10863    }
10864    (out, timed_out_pids)
10865}
10866
10867/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10868/// section (`report::run`) and in the `merge` event this node records — for
10869/// the operator to run by hand.
10870///
10871/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10872/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10873/// merge commits", or "require linear history") rejects the push a `--no-ff`
10874/// merge would produce, which is exactly the guidance this function replaces.
10875/// `message`'s first line becomes the squash commit's subject, matching the
10876/// note `report::run` prints alongside this command — see that function for
10877/// why an explicit subject is not optional there.
10878fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10879    let repo = repo.display();
10880    match style {
10881        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10882        MergeStyle::Squash => {
10883            // The subject sits inside double quotes, and a title an agent
10884            // wrote may carry the characters that break out of them.
10885            let subject = message
10886                .lines()
10887                .next()
10888                .unwrap_or(branch)
10889                .replace(['\\', '"', '$', '`'], "");
10890            format!(
10891                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10892            )
10893        }
10894        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10895    }
10896}
10897
10898/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10899/// under the hood, rejects a `title` over 256 characters and the whole
10900/// command fails — no PR at all, for a run whose body was otherwise fine
10901/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10902/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10903/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10904/// this bound. It is a margin, not a guarantee — a title packed
10905/// with multi-unit characters could still in principle land close to the
10906/// edge, but a real task title's occasional emoji or accented letter fits
10907/// comfortably inside it.
10908const PR_TITLE_MAX: usize = 240;
10909
10910/// A pull request title from the opening line of `text`, or `None` when that
10911/// line is not English (GitHub text is) or has no letters.
10912///
10913/// A line within `max` is kept as is. A longer one is cut at the end of its
10914/// first sentence when that falls inside `max`, else at a word boundary with a
10915/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10916/// make every merely-truncated title look non-English). The language check
10917/// runs on the kept text before any mark is added, so only what GitHub will
10918/// show is judged: an English opening followed by non-ASCII far past the cut
10919/// still passes.
10920fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10921    let line = queue::first_line(text)?;
10922    let chars: Vec<char> = line.chars().collect();
10923    let (kept, mark) = if chars.len() <= max {
10924        (line.to_owned(), "")
10925    } else if let Some(end) = sentence_end(&chars, max) {
10926        (chars[..end].iter().collect::<String>(), "")
10927    } else {
10928        let room = max.saturating_sub(3);
10929        // Cut at the last space inside the room; when the char just past the
10930        // room is a space the room already ends on a word.
10931        let cut = if chars[room].is_whitespace() {
10932            room
10933        } else {
10934            chars[..room]
10935                .iter()
10936                .rposition(|c| c.is_whitespace())
10937                .unwrap_or(room)
10938        };
10939        let head: String = chars[..cut].iter().collect();
10940        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10941        (head.to_owned(), "...")
10942    };
10943    if kept.is_empty() {
10944        return None;
10945    }
10946    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10947        return None;
10948    }
10949    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10950        return None;
10951    }
10952    Some(format!("{kept}{mark}"))
10953}
10954
10955/// The char length of the first sentence of `chars` when it ends within `max`
10956/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10957/// abbreviations so `e.g. foo` does not end a title early.
10958fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10959    const MIN: usize = 20;
10960    for i in MIN..max.min(chars.len()) {
10961        if !matches!(chars[i], '.' | '!' | '?') {
10962            continue;
10963        }
10964        let Some(&next) = chars.get(i + 1) else {
10965            continue;
10966        };
10967        if !next.is_whitespace() {
10968            continue;
10969        }
10970        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10971        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10972            continue;
10973        }
10974        let word: String = chars[..i]
10975            .iter()
10976            .rev()
10977            .take_while(|c| !c.is_whitespace())
10978            .collect::<Vec<_>>()
10979            .into_iter()
10980            .rev()
10981            .collect();
10982        let word = word.to_ascii_lowercase();
10983        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10984            continue;
10985        }
10986        let end = chars[..i]
10987            .iter()
10988            .rposition(|c| !c.is_whitespace())
10989            .map_or(i, |p| p + 1);
10990        return Some(end);
10991    }
10992    None
10993}
10994
10995/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10996/// change: a title and a body describing what was *implemented*, not the task
10997/// that asked for it. A task reads as a request; a reader of the merged
10998/// history wants the change.
10999struct PrMessage {
11000    title: String,
11001    body: String,
11002}
11003
11004impl PrMessage {
11005    /// Title, blank line, body. The first line is the squash/merge commit
11006    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11007    /// has to stay one sensible line.
11008    fn commit_message(&self) -> String {
11009        format!("{}\n\n{}", self.title, self.body)
11010    }
11011}
11012
11013/// The text after a leading `TITLE:` (any case) on `line`.
11014fn title_marker(line: &str) -> Option<&str> {
11015    let line = line.trim();
11016    let head = line.get(..6)?;
11017    head.eq_ignore_ascii_case("title:")
11018        .then(|| line[6..].trim())
11019}
11020
11021/// The implementer's own one-line title: the `TITLE:` line the implement
11022/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11023/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11024/// source, and a title that says as much is refused here too.
11025fn summary_title(summary: &str) -> Option<String> {
11026    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11027    let raw = title_marker(first)?;
11028    if raw.is_empty() {
11029        return None;
11030    }
11031    let title = queue::title_from(raw, PR_TITLE_MAX);
11032    let lower = title.to_ascii_lowercase();
11033    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11034        return None;
11035    }
11036    Some(title)
11037}
11038
11039/// How `open_review`'s instruction begins; see [`landing_title`].
11040const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11041
11042/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11043pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11044
11045/// Does this run review work that already existed, rather than implement a
11046/// task? Runs recorded before `reviewed_commits` existed carry only the
11047/// candidate marker.
11048pub(crate) fn is_review_run(state: &RunState) -> bool {
11049    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11050}
11051
11052/// The title of a review-only run: the subject of the oldest commit under
11053/// review. Later commits are usually fixups, and `instruction` is the review
11054/// prompt, which says nothing about the change. GitHub text is English, so a
11055/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11056fn review_title(state: &RunState) -> Option<String> {
11057    english_subject(
11058        state.reviewed_commits.as_ref()?.first()?,
11059        state.config.graph.github_english(),
11060    )
11061}
11062
11063/// `raw` as a pull request title, or `None` when it is blank, not English
11064/// (GitHub text is), or one of magi's own candidate commit subjects.
11065fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11066    let raw = raw.trim();
11067    if raw.is_empty() {
11068        return None;
11069    }
11070    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11071        return None;
11072    }
11073    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11074        return None;
11075    }
11076    let title = queue::title_from(raw, PR_TITLE_MAX);
11077    let lower = title.to_ascii_lowercase();
11078    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11079        return None;
11080    }
11081    Some(title)
11082}
11083
11084/// What a review-only run's branch says about itself, read at the moment the
11085/// pull request is opened.
11086#[derive(Debug, Clone, PartialEq, Eq)]
11087struct BranchFacts {
11088    /// `(subject, body)` of each commit, oldest first.
11089    commits: Vec<(String, String)>,
11090    /// Trimmed `git diff --stat`.
11091    stat: String,
11092}
11093
11094/// Longest diff stat shown: this many file lines plus the summary line.
11095const STAT_FILE_LINES: usize = 25;
11096/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11097const COMMITS_MAX_CHARS: usize = 20_000;
11098
11099/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11100/// say or finds nothing, so the caller falls back to what the run recorded.
11101async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11102    let commits = git::commit_log(repo, base, branch).await.ok()?;
11103    if commits.is_empty() {
11104        return None;
11105    }
11106    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11107    let lines: Vec<&str> = stat.lines().collect();
11108    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11109        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11110        let more = format!(" ... {omitted} more file(s)");
11111        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11112        kept.push(&more);
11113        kept.push(lines[lines.len() - 1]);
11114        kept.join("\n")
11115    } else {
11116        lines.join("\n")
11117    };
11118    Some(BranchFacts { commits, stat })
11119}
11120
11121/// Defang what would break the surrounding markdown: a closing `</details>`
11122/// and a code fence.
11123fn markdown_safe(text: &str) -> String {
11124    text.replace("</details>", "&lt;/details&gt;")
11125        .replace("\x60\x60\x60", "~~~")
11126}
11127
11128fn neutral_title(state: &RunState, winner: char) -> String {
11129    format!(
11130        "chore: land candidate {} of run {}",
11131        winner.to_ascii_uppercase(),
11132        state.id
11133    )
11134}
11135
11136/// Reuse the graph's bounded fixer without starting another graph or driver.
11137/// Copy the working state back even on errors so spent rounds remain visible.
11138pub(crate) async fn repair_landing_gate(
11139    state: &mut RunState,
11140    winner: &Candidate,
11141    outcomes: &[CommandOutcome],
11142) -> Result<GateFix> {
11143    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11144        return Ok(GateFix::Stop);
11145    }
11146    let mut runner = Runner {
11147        roles: state.config.resolve_roles()?,
11148        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11149        state: state.clone(),
11150        pause: Pause::new(),
11151        interrupt: Pause::new(),
11152    };
11153    let result = runner.gate_fix_round(winner, outcomes).await;
11154    *state = runner.state;
11155    result
11156}
11157
11158/// Recheck a repair's e2e requirement across failures, resource deferrals and
11159/// process interruptions. A successful result only applies to its exact SHA.
11160pub(crate) async fn check_gate_repair_e2e(
11161    state: &mut RunState,
11162    winner: &Candidate,
11163    wait: &crate::repo_lock::WaitPolicy,
11164) -> Result<GateFix> {
11165    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11166        return Ok(GateFix::Retry);
11167    }
11168    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11169    if state
11170        .gate_repair_e2e
11171        .as_ref()
11172        .is_some_and(|r| r.head == head && r.passed)
11173    {
11174        return Ok(GateFix::Retry);
11175    }
11176    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11177        head: head.clone(),
11178        passed: false,
11179        outcomes: Vec::new(),
11180    });
11181    state.save()?;
11182    let commands = state.config.verify.e2e.clone();
11183    let outcomes = if commands.is_empty() {
11184        Vec::new()
11185    } else {
11186        let shell = state.config.shell();
11187        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11188        let cache_dir = state.config.cache_dir();
11189        let (outcomes, _) = with_cache_lease(
11190            state,
11191            cache_dir.as_deref(),
11192            "e2e",
11193            "e2e",
11194            &winner.worktree,
11195            &head,
11196            timeout,
11197            "gate repair e2e",
11198            wait,
11199            |state, budget| async move {
11200                run_e2e_with_retry(
11201                    state,
11202                    &shell,
11203                    &commands,
11204                    &winner.worktree,
11205                    budget,
11206                    "gate repair e2e",
11207                )
11208                .await
11209            },
11210        )
11211        .await?;
11212        outcomes
11213    };
11214    let deferred = verify_inconclusive(&outcomes);
11215    let passed = outcomes.iter().all(CommandOutcome::ok)
11216        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11217    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11218        head,
11219        passed,
11220        outcomes,
11221    });
11222    state.save()?;
11223    if deferred {
11224        Ok(GateFix::Defer)
11225    } else if !passed {
11226        state.event(
11227            "gate",
11228            "verify.e2e failed after gate repair; nothing published",
11229        );
11230        Ok(GateFix::Stop)
11231    } else {
11232        Ok(GateFix::Retry)
11233    }
11234}
11235
11236/// Run the same configured gate for a landing head and a fresh graph.
11237pub(crate) async fn run_landing_gate(
11238    state: &mut RunState,
11239    winner: &Candidate,
11240    wait: &crate::repo_lock::WaitPolicy,
11241) -> Result<Vec<CommandOutcome>> {
11242    let shell = state.config.shell();
11243    let gate_commands = state.config.verify.gate.clone();
11244    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11245    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11246        Vec::new()
11247    } else {
11248        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11249        let cache_dir = state.config.cache_dir();
11250        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11251        let (outcomes, _) = with_cache_lease(
11252            state,
11253            cache_dir.as_deref(),
11254            "gate",
11255            "gate",
11256            &winner.worktree,
11257            &head,
11258            timeout,
11259            "final gate",
11260            wait,
11261            |state, budget| {
11262                let shell = shell.clone();
11263                let gate_commands = gate_commands.clone();
11264                let winner = winner.clone();
11265                async move {
11266                    Runner::run_pre_gate(state, &winner).await;
11267                    let (outcomes, timed_out_pids) = run_commands(
11268                        state,
11269                        "gate",
11270                        "gate",
11271                        0,
11272                        &shell,
11273                        &gate_commands,
11274                        &winner.worktree,
11275                        budget,
11276                    )
11277                    .await;
11278                    (outcomes, false, timed_out_pids)
11279                }
11280            },
11281        )
11282        .await?;
11283        outcomes
11284    };
11285    if outcomes.is_empty() {
11286        // Nothing configured to check — distinct from every other
11287        // silence in this run's event log, since an empty `gate` alone
11288        // no longer says whether the gate ran at all (see
11289        // `RunState::gate_ran`'s own doc).
11290        state.event("gate", "no gate commands configured; not tested");
11291    }
11292    for o in &outcomes {
11293        state.event(
11294            "gate",
11295            format!(
11296                "`{}` -> {}",
11297                o.command,
11298                if o.ok() {
11299                    "pass".to_owned()
11300                } else {
11301                    format!(
11302                        "FAIL ({:?})\n{}",
11303                        o.code,
11304                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11305                    )
11306                }
11307            ),
11308        );
11309    }
11310    Ok(outcomes)
11311}
11312
11313/// The pull request title to hand to `land::merge_subject`. A review-only run
11314/// opened by an earlier build titled its pull request with the review prompt;
11315/// that title is dropped (empty, so the fallback applies) rather than landed.
11316/// Any other title, including an operator's rename, passes through untouched,
11317/// and so does every title of a run that implements a task.
11318pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11319    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11320        ""
11321    } else {
11322        pr_title
11323    }
11324}
11325
11326/// What the squash subject falls back to when the pull request title is empty
11327/// or candidate-shaped: for a review-only run the derived title, never the
11328/// review prompt held in `instruction`.
11329pub fn landing_subject_source(state: &RunState) -> String {
11330    if is_review_run(state) {
11331        let winner = state.candidates.first().map_or('A', |c| c.label);
11332        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11333    }
11334    state.instruction.clone()
11335}
11336
11337/// `summary` without its `TITLE:` line, which the pull request title already
11338/// carries.
11339fn summary_without_title(summary: &str) -> String {
11340    let mut lines = summary.trim().lines().peekable();
11341    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11342        lines.next();
11343    }
11344    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11345}
11346
11347/// The pull request title and body for the winning candidate.
11348///
11349/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11350/// the task's own opening line via [`queue::title_from`] when there is none.
11351/// `state.instruction` can open with blank lines (`task_text` only rejects a
11352/// body that is blank *entirely*), which `title_from` skips.
11353///
11354/// Publishing uses an independent description generated from the final diff
11355/// and structured evidence. Internal summaries and reports remain local.
11356/// Non-publishing previews may display the original task and candidate summary.
11357/// The footer retains the run and candidate provenance.
11358#[cfg(test)]
11359fn pr_message(state: &RunState, winner: char) -> PrMessage {
11360    pr_message_with(state, winner, None)
11361}
11362
11363/// [`pr_message`] with what the branch of a review-only run says about itself.
11364/// `facts` is ignored for a run that implements a task.
11365#[cfg(test)]
11366fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11367    let raw = pr_message_raw(
11368        state,
11369        winner,
11370        facts,
11371        crate::pr_outcome::FinalHead::Unreported,
11372        ChangeNote::NotAsked,
11373    );
11374    let id = crate::scrub::Identity::current();
11375    PrMessage {
11376        title: crate::scrub::scrub(&raw.title, &id),
11377        body: crate::scrub::scrub(&raw.body, &id),
11378    }
11379}
11380
11381/// What the public description may say about the change itself.
11382#[derive(Debug, Clone, Copy)]
11383enum ChangeNote<'a> {
11384    /// A local preview or non-publishing merge mode: no public generation needed.
11385    NotAsked,
11386    /// A summary written from the final diff.
11387    Summary(&'a crate::run::ChangeSummary),
11388    /// A summary was asked for and could not be produced; the text says why.
11389    Unavailable(&'a str),
11390}
11391
11392fn pr_message_raw(
11393    state: &RunState,
11394    winner: char,
11395    facts: Option<&BranchFacts>,
11396    head: crate::pr_outcome::FinalHead<'_>,
11397    change: ChangeNote<'_>,
11398) -> PrMessage {
11399    if !matches!(change, ChangeNote::NotAsked) {
11400        let (title, mut body) = match change {
11401            ChangeNote::Summary(c) => (
11402                c.title.clone(),
11403                format!("## Summary\n\n{}\n\n", c.body.trim()),
11404            ),
11405            ChangeNote::Unavailable(why) => (
11406                neutral_title(state, winner),
11407                format!(
11408                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11409                ),
11410            ),
11411            ChangeNote::NotAsked => unreachable!(),
11412        };
11413        body.push_str(&public_pr_evidence(state, head));
11414        body.push_str(&format!(
11415            "\n---\nmagi:run/{} magi:candidate-{}\n",
11416            state.id,
11417            winner.to_ascii_lowercase()
11418        ));
11419        return PrMessage { title, body };
11420    }
11421    pr_preview(state, winner, facts, head)
11422}
11423
11424/// Append public facts from the run records after every prose generation,
11425/// including posting-gate rewrites. The agent cannot drop this provenance.
11426fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11427    let mut body = crate::pr_outcome::review_rounds(state);
11428    body.push_str(&crate::pr_outcome::final_verification(state, head));
11429    let open = state.last_round_findings();
11430    if !open.is_empty() {
11431        body.push_str("## Open review findings\n\n");
11432        for f in open {
11433            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11434        }
11435        body.push('\n');
11436    }
11437    if let Some(c) = state.change_summary.as_ref()
11438        && head == crate::pr_outcome::FinalHead::At(&c.head)
11439    {
11440        body.push_str(&format!(
11441            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11442            c.base, c.head
11443        ));
11444    }
11445    body
11446}
11447
11448/// Local-only preview of the operator's task and candidate summary.
11449fn pr_preview(
11450    state: &RunState,
11451    winner: char,
11452    facts: Option<&BranchFacts>,
11453    head: crate::pr_outcome::FinalHead<'_>,
11454) -> PrMessage {
11455    let summary = state
11456        .candidates
11457        .iter()
11458        .find(|c| c.label == winner)
11459        .map(|c| c.summary.as_str())
11460        .unwrap_or_default();
11461    // The fallback is the operator's own words and may not be English; GitHub
11462    // text always is, so a non-English task gets a neutral title instead.
11463    let review = is_review_run(state);
11464    let title = if review {
11465        facts
11466            .and_then(|f| {
11467                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11468            })
11469            .or_else(|| review_title(state))
11470            .or_else(|| {
11471                state
11472                    .candidates
11473                    .iter()
11474                    .find(|c| c.label == winner)
11475                    .filter(|c| !c.branch.starts_with("magi/"))
11476                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11477            })
11478            .unwrap_or_else(|| neutral_title(state, winner))
11479    } else {
11480        summary_title(summary).unwrap_or_else(|| {
11481            english_title(
11482                &state.instruction,
11483                PR_TITLE_MAX,
11484                state.config.graph.github_english(),
11485            )
11486            .unwrap_or_else(|| neutral_title(state, winner))
11487        })
11488    };
11489
11490    let mut body = String::new();
11491    let what = summary_without_title(summary);
11492    if !what.is_empty() {
11493        body.push_str("## Summary\n\n");
11494        body.push_str(&what);
11495        body.push_str("\n\n");
11496    }
11497
11498    // Review history uses structured facts across all rounds, never reports.
11499    body.push_str(&crate::pr_outcome::review_rounds(state));
11500    body.push_str(&crate::pr_outcome::final_verification(state, head));
11501    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11502
11503    let open = state.open_findings();
11504    if !open.is_empty() {
11505        body.push_str("## Open review findings\n\n");
11506        for f in &open {
11507            body.push_str(&format!(
11508                "- `{}` [{:?}] {}\n",
11509                f.id, f.severity, "Context retained in local review records."
11510            ));
11511        }
11512        body.push('\n');
11513    }
11514
11515    if let Some(fix) = fix
11516        && !fix.rejected.is_empty()
11517    {
11518        body.push_str("## Declined by the fixer\n\n");
11519        for r in &fix.rejected {
11520            body.push_str(&format!(
11521                "- `{}`: {}\n",
11522                r.id, "Reason retained in local review records."
11523            ));
11524        }
11525        body.push('\n');
11526    }
11527
11528    if review {
11529        // The review prompt is not the task; list what the branch carries.
11530        body.push_str("## Commits under review\n\n");
11531        if let Some(facts) = facts {
11532            let mut left = COMMITS_MAX_CHARS;
11533            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11534                let mut entry = format!("- {}\n", markdown_safe(subject));
11535                for l in markdown_safe(text).lines() {
11536                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11537                }
11538                if left == 0 {
11539                    body.push_str(&format!(
11540                        "- ... {} more commit(s)\n",
11541                        facts.commits.len() - i
11542                    ));
11543                    break;
11544                }
11545                if entry.len() > left {
11546                    // Even the first commit is cut: one huge body must not
11547                    // push the whole description past GitHub's limit.
11548                    let mut end = left;
11549                    while !entry.is_char_boundary(end) {
11550                        end -= 1;
11551                    }
11552                    entry.truncate(end);
11553                    entry.push_str("\n  ... (truncated)\n");
11554                    left = 0;
11555                } else {
11556                    left -= entry.len();
11557                }
11558                body.push_str(&entry);
11559            }
11560            if !facts.stat.trim().is_empty() {
11561                body.push_str(&format!(
11562                    "\n## Diff stat\n\n```\n{}\n```\n",
11563                    markdown_safe(facts.stat.trim())
11564                ));
11565            }
11566        } else {
11567            match &state.reviewed_commits {
11568                Some(subjects) => {
11569                    for s in subjects {
11570                        body.push_str(&format!("- {}\n", s.trim()));
11571                    }
11572                }
11573                None => {
11574                    // An older run kept only the prompt, with the commit list
11575                    // after its first paragraph.
11576                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11577                    body.push_str(rest.trim());
11578                    body.push('\n');
11579                }
11580            }
11581        }
11582    } else {
11583        let task = state.instruction.trim();
11584        let task = if task.is_empty() {
11585            "(empty task)"
11586        } else {
11587            task
11588        };
11589        body.push_str(&format!(
11590            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11591            task.replace("</details>", "&lt;/details&gt;")
11592        ));
11593    }
11594
11595    body.push_str(&format!(
11596        "\n---\nmagi:run/{} magi:candidate-{}\n",
11597        state.id,
11598        winner.to_ascii_lowercase()
11599    ));
11600
11601    PrMessage { title, body }
11602}
11603
11604/// Source imports have already been prepared from structured inputs. Legacy
11605/// inferred references must never instruct a seat to reconstruct those imports.
11606fn seeded_instruction(state: &RunState) -> String {
11607    let mut inputs = Vec::new();
11608    if let Some(start) = &state.sources.start_from {
11609        inputs.push(format!(
11610            "Start from `{}` at `{}` (including ancestry).",
11611            start.token, start.sha
11612        ));
11613    }
11614    for pick in &state.sources.pick_commits {
11615        inputs.push(format!(
11616            "Pick `{}` at `{}` (single commit).",
11617            pick.token, pick.sha
11618        ));
11619    }
11620    if inputs.is_empty() {
11621        state.instruction.clone()
11622    } else {
11623        format!(
11624            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11625            state.instruction,
11626            inputs.join("\n")
11627        )
11628    }
11629}
11630
11631/// Does the winner have no commits ahead of the base it would land on?
11632/// Any failure to find out reads as "not empty": the merge then behaves as it
11633/// always did rather than refusing on a guess.
11634async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11635    let base = &state.base_branch;
11636    let mut against = base.clone();
11637    if mode == MergeMode::Pr {
11638        // A pull request lands on the remote's base, never the local branch
11639        // of the same name: if that cannot be read, "not empty" is the safe
11640        // answer.
11641        let remote = &state.config.merge.remote;
11642        let tracking = format!("{remote}/{base}");
11643        let fetched = git::fetch(repo, remote, base).await;
11644        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11645            against = tracking;
11646        } else {
11647            return false;
11648        }
11649    }
11650    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11651}
11652
11653/// Why nothing was opened for an empty winner, with what the task's own
11654/// references resolved to.
11655fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11656    let mut detail = format!(
11657        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11658         nothing to open a pull request for"
11659    );
11660    detail.push_str(
11661        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11662    );
11663    detail
11664}
11665
11666/// What the `Pr` merge does once it knows whether the branch already has an
11667/// open pull request.
11668#[derive(Debug, PartialEq, Eq)]
11669enum PrPlan {
11670    Create,
11671    Adopt { url: String, title: String },
11672    Stop(String),
11673}
11674
11675/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11676/// a failed lookup -> stop with the real reason. Never guesses.
11677fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11678    match found {
11679        Ok(land::OpenPr::None) => PrPlan::Create,
11680        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11681        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11682            "several open pull requests exist for this branch, not picking one: {}",
11683            urls.join(" ")
11684        )),
11685        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11686    }
11687}
11688
11689/// `gh pr create`, returning the PR url.
11690pub(crate) async fn gh_pr_create(
11691    cwd: &Path,
11692    base: &str,
11693    head: &str,
11694    title: &str,
11695    body: &str,
11696) -> Result<String> {
11697    let out = tokio::process::Command::new("gh")
11698        .args([
11699            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11700        ])
11701        .current_dir(cwd)
11702        .quiet()
11703        .stdin(std::process::Stdio::null())
11704        .output()
11705        .await
11706        .context("spawn gh")?;
11707    if out.status.success() {
11708        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11709    } else {
11710        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11711    }
11712}
11713
11714/// Never force deletion of uncommitted work, including unknown legacy trees.
11715async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11716    match std::fs::symlink_metadata(path) {
11717        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11718            // A previous cleanup may have removed the tree before a branch
11719            // deletion or state save failed. Unregister it if still present.
11720            git::worktree_remove_clean(repo, path).await?;
11721            return Ok(());
11722        }
11723        Err(e) => return Err(e.into()),
11724        Ok(_) => {}
11725    }
11726    if !git::is_clean(path).await? {
11727        bail!(
11728            "worktree has uncommitted work; refusing removal: {}",
11729            path.display()
11730        );
11731    }
11732    if !git::worktree_remove_clean(repo, path).await? {
11733        bail!("worktree removal refused: {}", path.display());
11734    }
11735    Ok(())
11736}
11737
11738/// Tear a run's worktrees and branches down.
11739///
11740/// `home` is where the updated `run.json` is saved (via
11741/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11742/// a housekeeping pass already has its own honest `home` handed to it, and
11743/// falling through to the global here would write back through whichever
11744/// directory some other process or test pinned into that `OnceLock` first,
11745/// not the one the caller actually resolved its `runs` and `state` from.
11746pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11747    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11748        bail!("refusing fold: {reason}");
11749    }
11750    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11751        bail!("candidate preservation incomplete; refusing cleanup");
11752    }
11753    // Serialize ordinary folding with the same starts/resumes as target-only
11754    // eviction. Otherwise a second janitor could remove the whole worktree
11755    // around a live output borrower even though target eviction was fenced.
11756    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11757    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11758    let _driver = crate::driver::maintenance(home, &state.id)?;
11759    let mut output_guards = Vec::new();
11760    for entry in std::fs::read_dir(state.worktree_root())
11761        .into_iter()
11762        .flatten()
11763        .flatten()
11764    {
11765        if entry.path().join(".git").is_file() {
11766            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11767        }
11768    }
11769    let repo = state.repo.clone();
11770    // CLI, Web and automatic cleanup all enter here, independently of
11771    // Runner::fold_losers. Protect deletion and shared config release too.
11772    let wait = repository_wait(state, None);
11773    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11774        Ok(lease) => lease,
11775        Err(e) => {
11776            state.save_under(home)?;
11777            return Err(e);
11778        }
11779    };
11780    // Confirmed merges dispose of their owned sources after durable evidence
11781    // capture. Preflight every source before removing anything.
11782    let _source_outputs = if state.status == RunStatus::Merged
11783        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11784    {
11785        verify_merged_fold_sources(state, drop_winner, home).await?
11786    } else {
11787        Vec::new()
11788    };
11789    let root = state.worktree_root();
11790    let winner = state.tally.as_ref().map(|t| t.winner);
11791    let mut removed = Vec::new();
11792
11793    for i in 0..state.candidates.len() {
11794        let c = state.candidates[i].clone();
11795        let is_winner = Some(c.label) == winner;
11796        if is_winner && !drop_winner {
11797            continue;
11798        }
11799        let existed = c.worktree.exists();
11800        // A vanished directory can still have a Git registration that keeps
11801        // its branch checked out. Remove that registration before the branch.
11802        remove_preserved_worktree(&repo, &c.worktree).await?;
11803        if existed {
11804            removed.push(c.worktree.to_string_lossy().into_owned());
11805        }
11806        // A branch handed to a later run (and its pull request) is not this
11807        // run's to delete.
11808        let handed_over = state.released_branches.contains(&c.branch);
11809        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11810            if !git::branch_delete(&repo, &c.branch).await? {
11811                bail!("candidate branch removal refused");
11812            }
11813            removed.push(c.branch.clone());
11814        }
11815        state.candidates[i].folded = true;
11816    }
11817
11818    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11819        let path = name.path();
11820        let keep = !drop_winner
11821            && winner.is_some_and(|w| {
11822                path.file_name()
11823                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11824            });
11825        if keep {
11826            continue;
11827        }
11828        remove_preserved_worktree(&repo, &path).await?;
11829        removed.push(path.to_string_lossy().into_owned());
11830    }
11831
11832    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11833    // judge worktrees, so once the loop above has cleared all of them out,
11834    // the parent is a bare directory nobody else was ever going to remove -
11835    // git only ever managed what was inside it. Left alone, one of these
11836    // accumulates per fully-folded run; the operator's own machine had 74.
11837    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11838    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11839    // so does anything a run never claimed that happens to share the bay.
11840    remove_if_empty(&root);
11841
11842    if state.enabled_worktree_config && drop_winner {
11843        // A release, not a raw disable: some sibling run in this repository
11844        // may still hold its own reference (see `git::acquire_worktree_config`),
11845        // and only the last release actually turns the setting back off.
11846        git::release_worktree_config(&repo).await.ok();
11847        state.enabled_worktree_config = false;
11848    }
11849    state.save_under(home)?;
11850    Ok(removed)
11851}
11852
11853async fn verify_merged_fold_sources(
11854    state: &RunState,
11855    drop_winner: bool,
11856    home: &Path,
11857) -> Result<Vec<std::fs::File>> {
11858    let winner = state.tally.as_ref().map(|t| t.winner);
11859    let mut heads = BTreeSet::new();
11860    for candidate in &state.candidates {
11861        if Some(candidate.label) == winner && !drop_winner {
11862            continue;
11863        }
11864        anyhow::ensure!(
11865            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11866            "candidate recovery source is outside the inventoried run root: {}",
11867            candidate.worktree.display()
11868        );
11869        match std::fs::symlink_metadata(&candidate.worktree) {
11870            Ok(_) => {}
11871            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11872            Err(error) => return Err(error.into()),
11873        }
11874        if !state.released_branches.contains(&candidate.branch)
11875            && git::branch_exists(&state.repo, &candidate.branch).await?
11876        {
11877            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11878        }
11879    }
11880    let mut cargo_guards = Vec::new();
11881    let entries = match std::fs::read_dir(state.worktree_root()) {
11882        Ok(entries) => Some(entries),
11883        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11884        Err(error) => return Err(error.into()),
11885    };
11886    for entry in entries.into_iter().flatten() {
11887        let path = entry?.path();
11888        let metadata = std::fs::symlink_metadata(&path)?;
11889        anyhow::ensure!(
11890            metadata.is_dir() && !metadata.file_type().is_symlink(),
11891            "non-directory or symlink recovery source retained: {}",
11892            path.display()
11893        );
11894        if !drop_winner
11895            && winner.is_some_and(|w| {
11896                path.file_name()
11897                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11898            })
11899        {
11900            continue;
11901        }
11902        anyhow::ensure!(
11903            path.join(".git").is_file(),
11904            "unregistered recovery source retained: {}",
11905            path.display()
11906        );
11907        anyhow::ensure!(
11908            git::git(
11909                &path,
11910                &[
11911                    "status",
11912                    "--porcelain",
11913                    "--untracked-files=all",
11914                    "--ignored",
11915                    "--",
11916                    ".",
11917                    ":(exclude)target"
11918                ]
11919            )
11920            .await?
11921            .is_empty(),
11922            "source changes or ignored recovery files retained: {}",
11923            path.display()
11924        );
11925        match std::fs::symlink_metadata(path.join("target")) {
11926            Ok(_) => {
11927                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11928            }
11929            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11930            Err(error) => return Err(error.into()),
11931        }
11932        for checkpoint in [
11933            "rebase-merge",
11934            "rebase-apply",
11935            "sequencer",
11936            "CHERRY_PICK_HEAD",
11937            "MERGE_HEAD",
11938            "REBASE_HEAD",
11939            "REVERT_HEAD",
11940            "BISECT_LOG",
11941            "AUTO_MERGE",
11942            "MERGE_MSG",
11943        ] {
11944            let location = git::git(
11945                &path,
11946                &[
11947                    "rev-parse",
11948                    "--path-format=absolute",
11949                    "--git-path",
11950                    checkpoint,
11951                ],
11952            )
11953            .await?;
11954            anyhow::ensure!(
11955                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11956                if e.kind() == std::io::ErrorKind::NotFound),
11957                "Git recovery checkpoint retained: {checkpoint}"
11958            );
11959        }
11960        heads.insert(git::rev_parse(&path, "HEAD").await?);
11961    }
11962    crate::fold_evidence::preserve(state, home, &heads).await?;
11963
11964    Ok(cargo_guards)
11965}
11966
11967/// Remove `dir` if it exists and has nothing in it.
11968///
11969/// Best-effort and silent by design: a directory that is not empty (a run
11970/// whose winner is still parked there, a stray file some other process left)
11971/// is exactly the case this must refuse, and a directory that is already gone
11972/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11973/// already refuses a non-empty directory, so the emptiness check below is
11974/// belt, not suspenders - it is what keeps this from ever attempting the
11975/// removal in the case that matters, rather than trusting `remove_dir`'s
11976/// error path to have no side effects if it ever changed.
11977fn remove_if_empty(dir: &Path) {
11978    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11979        std::fs::remove_dir(dir).ok();
11980    }
11981}
11982
11983/// Severity of the worst open finding in the last review round, for reporting.
11984pub fn worst_open(state: &RunState) -> Option<Severity> {
11985    state
11986        .reviews
11987        .last()?
11988        .reviews
11989        .iter()
11990        .flat_map(|r| r.findings.iter())
11991        .map(|f| f.severity)
11992        .max()
11993}
11994
11995/// `[worktree] setup` hides its products per worktree, which needs
11996/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11997/// worktrees, like the hook's; every place that runs setup calls this first.
11998async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11999    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12000        git::acquire_worktree_config(repo).await?;
12001        state.enabled_worktree_config = true;
12002    }
12003    Ok(())
12004}
12005
12006#[cfg(test)]
12007mod tests {
12008    #[tokio::test]
12009    async fn launcher_base_fetch_requires_remote_success() {
12010        let tmp = tempfile::tempdir().unwrap();
12011        let repo = tmp.path().join("repo");
12012        let remote = tmp.path().join("remote");
12013        std::fs::create_dir(&repo).unwrap();
12014        std::fs::create_dir(&remote).unwrap();
12015        git::git(&remote, &["init", "--bare", "-b", "main"])
12016            .await
12017            .unwrap();
12018        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12019        git::git(
12020            &repo,
12021            &[
12022                "-c",
12023                "user.name=Fixture",
12024                "-c",
12025                "user.email=fixture@example.invalid",
12026                "commit",
12027                "--allow-empty",
12028                "-m",
12029                "fixture",
12030            ],
12031        )
12032        .await
12033        .unwrap();
12034        git::git(
12035            &repo,
12036            &["remote", "add", "origin", remote.to_str().unwrap()],
12037        )
12038        .await
12039        .unwrap();
12040        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12041        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12042        assert_eq!(
12043            resolve_base(&repo, "main", "origin", None)
12044                .await
12045                .unwrap()
12046                .commit,
12047            head
12048        );
12049        #[cfg(unix)]
12050        {
12051            let policy: crate::config::Coordination = toml::from_str(
12052                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12053            ).unwrap();
12054            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12055            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12056            let held = crate::repo_lock::SeatLease::operation_with(
12057                &repo,
12058                Some(&policy),
12059                &crate::repo_lock::WaitPolicy::default(),
12060            )
12061            .await
12062            .unwrap();
12063            assert!(held.is_some());
12064            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12065            tokio::pin!(fetch);
12066            assert!(
12067                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12068                    .await
12069                    .is_err()
12070            );
12071            assert!(
12072                !repo.join(".git/FETCH_HEAD").exists(),
12073                "no fetch before admission"
12074            );
12075            drop(held);
12076            let fetched = fetch.await.unwrap();
12077            assert_eq!(fetched.commit, head);
12078            assert_eq!(fetched.remote, "origin");
12079            assert_eq!(fetched.branch, "main");
12080        }
12081        // A surviving tracking ref is not successful fetch evidence.
12082        git::git(
12083            &repo,
12084            &[
12085                "remote",
12086                "set-url",
12087                "origin",
12088                tmp.path().join("missing").to_str().unwrap(),
12089            ],
12090        )
12091        .await
12092        .unwrap();
12093        let error = resolve_base(&repo, "main", "origin", None)
12094            .await
12095            .unwrap_err()
12096            .to_string();
12097        assert!(error.contains("cannot read origin/main"));
12098        assert!(error.contains("refusing to branch off"));
12099    }
12100
12101    #[test]
12102    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12103        let leaked = vec!["chore(deps): update a crate".to_owned()];
12104        let own = "fix(daemon): apply a chosen action";
12105        assert!(should_retitle("", own, &leaked));
12106        assert!(should_retitle(
12107            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12108            own,
12109            &leaked
12110        ));
12111        assert!(should_retitle(
12112            "chore: land candidate A of run 1",
12113            own,
12114            &leaked
12115        ));
12116        assert!(should_retitle(
12117            "magi: candidate A (uncommitted work)",
12118            own,
12119            &leaked
12120        ));
12121        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12122        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12123        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12124    }
12125
12126    #[test]
12127    fn pr_merge_plan_creates_adopts_or_stops() {
12128        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12129        assert_eq!(
12130            pr_merge_plan(Ok(land::OpenPr::One {
12131                url: "u".into(),
12132                title: "t".into()
12133            })),
12134            PrPlan::Adopt {
12135                url: "u".into(),
12136                title: "t".into()
12137            }
12138        );
12139        let PrPlan::Stop(many) =
12140            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12141        else {
12142            panic!("many must stop");
12143        };
12144        assert!(many.contains('a') && many.contains('b'));
12145        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12146            panic!("a failed lookup must stop");
12147        };
12148        assert!(err.contains("bad token"));
12149    }
12150
12151    use super::*;
12152    use crate::run::GateStatus;
12153    use std::collections::BTreeMap;
12154    use std::time::Duration;
12155
12156    fn conductor() -> AgentSpec {
12157        AgentSpec {
12158            id: "conductor".to_owned(),
12159            kind: crate::config::AgentKind::Command,
12160            model: None,
12161            command: vec!["true".to_owned()],
12162            extra_args: Vec::new(),
12163            env: BTreeMap::new(),
12164            prompt_delivery: None,
12165            max_turn_secs: None,
12166        }
12167    }
12168
12169    fn spec(id: &str) -> AgentSpec {
12170        AgentSpec {
12171            id: id.to_owned(),
12172            kind: crate::config::AgentKind::Command,
12173            model: None,
12174            command: vec!["true".to_owned()],
12175            extra_args: Vec::new(),
12176            env: BTreeMap::new(),
12177            prompt_delivery: None,
12178            max_turn_secs: None,
12179        }
12180    }
12181
12182    fn ids(xs: &[&str]) -> BTreeSet<String> {
12183        xs.iter().map(|s| (*s).to_owned()).collect()
12184    }
12185
12186    #[test]
12187    fn pick_successor_skips_an_agent_another_seat_holds() {
12188        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12189        let roster = [spec("a"), spec("b"), spec("c")];
12190        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12191        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12192    }
12193
12194    #[test]
12195    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12196        // The other seat started on c but was handed to d; c is free again.
12197        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12198        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12199        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12200        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12201        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12202    }
12203
12204    #[test]
12205    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12206        let roster = [spec("a"), spec("b")];
12207        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12208        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12209        let carried = ids(&["b"]);
12210        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12211        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12212    }
12213
12214    #[test]
12215    fn pick_successor_returns_none_without_any_untried_successor() {
12216        let roster = [spec("a"), spec("b")];
12217        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12218        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12219        let carried = ids(&["a"]);
12220        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12221    }
12222
12223    #[test]
12224    fn pick_successor_rescue_avoids_another_seats_occupant() {
12225        // b is a carried failure and free; a is held by the other seat.
12226        let roster = [spec("a"), spec("b"), spec("c")];
12227        let carried = ids(&["b", "c"]);
12228        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12229        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12230    }
12231
12232    #[test]
12233    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12234        let roster = [spec("a"), spec("b"), spec("c")];
12235        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12236        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12237    }
12238
12239    #[test]
12240    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12241        let roster = [spec("a"), spec("b"), spec("c")];
12242        let failed = ids(&["a", "b", "c"]);
12243        // Rescue looks at the whole roster, once per id, then runs out.
12244        let mut tried = ids(&["b"]);
12245        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12246        assert_eq!(first.id, "a");
12247        tried.insert(first.id.clone());
12248        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12249        assert_eq!(second.id, "c");
12250        tried.insert(second.id.clone());
12251        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12252    }
12253
12254    #[test]
12255    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12256        let roster = [spec("a"), spec("b")];
12257        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12258        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12259    }
12260
12261    #[test]
12262    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12263        let roster = [spec("a"), spec("b"), spec("c")];
12264        let h = SeatHistory {
12265            failed: ids(&["a"]),
12266            last_ok: Some("b".to_owned()),
12267            last_fail: None,
12268        };
12269        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12270        // A last answerer that left the roster, or later failed, is ignored.
12271        let gone = SeatHistory {
12272            last_ok: Some("zzz".to_owned()),
12273            ..h.clone()
12274        };
12275        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12276        let failed = SeatHistory {
12277            failed: ids(&["a", "b"]),
12278            last_ok: Some("b".to_owned()),
12279            last_fail: None,
12280        };
12281        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12282    }
12283
12284    #[test]
12285    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12286        let first = SeatState::new("review-1", "alpha", 7);
12287        let next = handover_seat("review-1", "gamma", 7);
12288        assert_ne!(first.claude_session, next.claude_session);
12289    }
12290
12291    #[test]
12292    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12293        let mut state = state_with_summary("x", "y");
12294        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12295        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12296        assert_ne!(a.claude_session, b.claude_session);
12297    }
12298
12299    #[test]
12300    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12301        let roster = [spec("a"), spec("b")];
12302        let h = SeatHistory {
12303            failed: ids(&["a", "b"]),
12304            ..SeatHistory::default()
12305        };
12306        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12307        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12308        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12309    }
12310
12311    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12312    // fallback loop depends on to terminate: it must walk forward from the
12313    // seat's own position, never restart at the front of the roster, and it
12314    // must never hand back an id already tried, however many times that id
12315    // happens to appear.
12316
12317    #[test]
12318    fn failure_signature_ignores_numbers_and_paths() {
12319        assert_eq!(
12320            failure_signature("exited with Some(2) and no usable output"),
12321            failure_signature("exited with Some(137) and no usable output")
12322        );
12323        assert_eq!(
12324            failure_signature("process exited with no usable output (exit code 2)"),
12325            failure_signature("process exited with no usable output (exit code 137)")
12326        );
12327        for (old, new) in [
12328            (
12329                "exited with Some(2) and no usable output",
12330                "process exited with no usable output (exit code 2)",
12331            ),
12332            (
12333                "exited with None and no usable output",
12334                "process exited with no usable output (exit code unavailable)",
12335            ),
12336        ] {
12337            let previous = FailClass::Other(failure_signature(old));
12338            let current = FailClass::Other(failure_signature(new));
12339            assert_eq!(previous, current, "resumption retains the failure class");
12340            assert!(!should_hand_over(Some(&previous), &current));
12341        }
12342        assert_eq!(
12343            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12344            failure_signature("cannot open /var/x.txt: denied")
12345        );
12346        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12347    }
12348
12349    #[test]
12350    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12351        let other = FailClass::Other("x".into());
12352        assert!(should_hand_over(None, &FailClass::Quota));
12353        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12354        assert!(should_hand_over(
12355            Some(&FailClass::Timeout),
12356            &FailClass::Timeout
12357        ));
12358        assert!(should_hand_over(None, &other));
12359        assert!(!should_hand_over(Some(&other), &other));
12360        assert!(should_hand_over(
12361            Some(&other),
12362            &FailClass::Other("y".into())
12363        ));
12364        // A quota or timeout in between ends the run of identical failures.
12365        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12366        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12367    }
12368
12369    #[test]
12370    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12371        let a = SeatState::new("judge-1", "alpha", 7);
12372        let b = handover_seat("judge-1", "beta", 7);
12373        assert_ne!(a.claude_session, b.claude_session);
12374        assert_eq!(b.turns, 0);
12375    }
12376
12377    #[test]
12378    fn a_timeout_is_classified_apart_from_other_failures() {
12379        assert_eq!(
12380            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12381            Some(FailClass::Timeout)
12382        );
12383        assert!(matches!(
12384            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12385            Some(FailClass::Other(_))
12386        ));
12387    }
12388
12389    #[test]
12390    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12391        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12392        let tried = BTreeSet::from(["beta".to_owned()]);
12393        // beta sits at index 1; the next candidate is gamma, never alpha —
12394        // which is very likely a different candidate slot's own agent.
12395        let next = next_untried_in_roster(&roster, 1, &tried);
12396        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12397    }
12398
12399    #[test]
12400    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12401        let roster = vec![spec("alpha"), spec("beta")];
12402        let tried = BTreeSet::from(["beta".to_owned()]);
12403        // beta is the roster's last entry: nothing follows it, and alpha —
12404        // earlier in the roster, almost certainly a different candidate
12405        // slot's own agent — must not be reached by wrapping back to it.
12406        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12407    }
12408
12409    #[test]
12410    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12411        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12412        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12413        // beta (index 1) and gamma (index 2, the only entry after it) have
12414        // both been tried; alpha (index 0) never has, but it comes before
12415        // beta's own position, so there is nothing further for this seat.
12416        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12417    }
12418
12419    #[test]
12420    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12421        let roster = vec![spec("a"), spec("a"), spec("b")];
12422        let tried = BTreeSet::from(["a".to_owned()]);
12423        let next = next_untried_in_roster(&roster, 0, &tried);
12424        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12425    }
12426
12427    #[test]
12428    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12429        let roster = vec![spec("a"), spec("b")];
12430        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12431        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12432    }
12433
12434    #[test]
12435    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12436        let dir = tempfile::tempdir().unwrap();
12437        let bay = dir.path().join("ffff");
12438
12439        // Not there yet: nothing to do, nothing to panic on.
12440        remove_if_empty(&bay);
12441        assert!(!bay.exists());
12442
12443        // Something still inside - the winner's worktree, or a stray file -
12444        // keeps the directory standing.
12445        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12446        remove_if_empty(&bay);
12447        assert!(bay.exists(), "non-empty directory must survive");
12448
12449        // Once the last entry is gone, so is the directory itself.
12450        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12451        remove_if_empty(&bay);
12452        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12453    }
12454
12455    // `round_is_clean` is the exact decision this task fixed: a round with a
12456    // seat that never answered must not read the same as a round every seat
12457    // actually reviewed. These are deterministic and process-free by design —
12458    // the equivalent end-to-end check (a real reviewer timing out under a
12459    // live graph run) is a genuine race against wall-clock contention, and a
12460    // spawn slow enough to blow even a generous budget under a loaded test
12461    // run must not turn this specific regression check flaky.
12462
12463    #[test]
12464    fn a_full_panel_that_found_nothing_is_clean() {
12465        assert!(round_is_clean(
12466            0,
12467            true,
12468            2,
12469            2,
12470            0,
12471            IncompleteReviewPolicy::Block
12472        ));
12473    }
12474
12475    #[test]
12476    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12477        assert!(!round_is_clean(
12478            0,
12479            true,
12480            1,
12481            2,
12482            0,
12483            IncompleteReviewPolicy::Block
12484        ));
12485    }
12486
12487    #[test]
12488    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12489        assert!(!round_is_clean(
12490            1,
12491            true,
12492            1,
12493            2,
12494            0,
12495            IncompleteReviewPolicy::Warn
12496        ));
12497    }
12498
12499    #[test]
12500    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12501        assert!(round_is_clean(
12502            0,
12503            true,
12504            1,
12505            2,
12506            0,
12507            IncompleteReviewPolicy::Warn
12508        ));
12509    }
12510
12511    #[test]
12512    fn a_full_panel_with_an_open_finding_is_not_clean() {
12513        assert!(!round_is_clean(
12514            1,
12515            true,
12516            2,
12517            2,
12518            0,
12519            IncompleteReviewPolicy::Block
12520        ));
12521    }
12522
12523    #[test]
12524    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12525        assert!(!round_is_clean(
12526            0,
12527            false,
12528            2,
12529            2,
12530            0,
12531            IncompleteReviewPolicy::Block
12532        ));
12533    }
12534
12535    // The stall this task closes: under the default `block` policy, a seat
12536    // missing only because it was rate limited must not force a wait for a
12537    // session limit that will not lift by the next round. `round_is_clean`
12538    // is where that quorum carve-out lives; the review loop around it never
12539    // changes what a reviewer's vote or a finding's severity means.
12540
12541    #[test]
12542    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12543        // 1 of 2 answered, and the one missing was quota'd — the exact
12544        // "review-2 rate limited (quota)" shape from the field report.
12545        assert!(round_is_clean(
12546            0,
12547            true,
12548            1,
12549            2,
12550            1,
12551            IncompleteReviewPolicy::Block
12552        ));
12553    }
12554
12555    #[test]
12556    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12557        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12558        // not a quota loss (`quota_missing` stays 0) — worth another try.
12559        assert!(!round_is_clean(
12560            0,
12561            true,
12562            1,
12563            2,
12564            0,
12565            IncompleteReviewPolicy::Block
12566        ));
12567    }
12568
12569    #[test]
12570    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12571        assert!(!round_is_clean(
12572            1,
12573            true,
12574            1,
12575            2,
12576            1,
12577            IncompleteReviewPolicy::Block
12578        ));
12579        assert!(!round_is_clean(
12580            0,
12581            false,
12582            1,
12583            2,
12584            1,
12585            IncompleteReviewPolicy::Block
12586        ));
12587    }
12588
12589    #[test]
12590    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12591        // Every seat quota'd, nobody answered: there is no panel to decide
12592        // on, so this must fall through to the existing block-and-retry
12593        // fallback rather than call an unreviewed patch clean.
12594        assert!(!round_is_clean(
12595            0,
12596            true,
12597            0,
12598            2,
12599            2,
12600            IncompleteReviewPolicy::Block
12601        ));
12602    }
12603
12604    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12605        CommandOutcome {
12606            command: "test".to_owned(),
12607            code,
12608            output_tail: String::new(),
12609            duration_ms: 0,
12610            resource_blocked,
12611        }
12612    }
12613
12614    #[test]
12615    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12616        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12617        assert!(
12618            !verify_inconclusive(&[outcome(Some(1), false)]),
12619            "an ordinary failure is still evidence about the patch"
12620        );
12621        assert!(verify_inconclusive(&[outcome(None, true)]));
12622        assert!(
12623            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12624            "one inconclusive outcome taints the whole batch"
12625        );
12626        assert!(!verify_inconclusive(&[]));
12627    }
12628
12629    #[tokio::test]
12630    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12631        // Alive for the first two checks, then dead - confirms the loop
12632        // actually re-polls rather than deciding once and sleeping out the
12633        // ceiling regardless.
12634        let calls = std::sync::atomic::AtomicUsize::new(0);
12635        let started = Instant::now();
12636        wait_for_pids_with(
12637            &[123],
12638            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12639            Duration::from_millis(5),
12640            Duration::from_secs(5),
12641        )
12642        .await;
12643        assert!(
12644            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12645            "must keep checking rather than deciding on the first answer"
12646        );
12647        assert!(
12648            started.elapsed() < Duration::from_secs(1),
12649            "must return the moment it is confirmed dead, not wait out the ceiling"
12650        );
12651    }
12652
12653    #[tokio::test]
12654    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12655        let started = Instant::now();
12656        wait_for_pids_with(
12657            &[123],
12658            |_| true, // never reports dead
12659            Duration::from_millis(5),
12660            Duration::from_millis(30),
12661        )
12662        .await;
12663        let elapsed = started.elapsed();
12664        assert!(
12665            elapsed >= Duration::from_millis(30),
12666            "must not give up before its own ceiling: {elapsed:?}"
12667        );
12668        assert!(
12669            elapsed < Duration::from_secs(1),
12670            "must not wait past its own ceiling either: {elapsed:?}"
12671        );
12672    }
12673
12674    #[tokio::test]
12675    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12676        let started = Instant::now();
12677        wait_for_pids_with(
12678            &[],
12679            |_| true,
12680            Duration::from_secs(5),
12681            Duration::from_secs(5),
12682        )
12683        .await;
12684        assert!(
12685            started.elapsed() < Duration::from_millis(200),
12686            "an empty pid list has nothing to confirm"
12687        );
12688    }
12689
12690    // `review_conclusion` is the exact decision the review hand-off task
12691    // fixed: a round budget spent (or a tree that stopped moving) must not
12692    // collapse into `Blocked` regardless of what verification actually
12693    // said. Deterministic and process-free for the same reason the
12694    // `round_is_clean` family above is.
12695    fn review_round(
12696        clean: bool,
12697        blocking: usize,
12698        answered: usize,
12699        expected: usize,
12700        progressed: bool,
12701        e2e_ok: bool,
12702    ) -> ReviewRound {
12703        ReviewRound {
12704            round: 1,
12705            head: "h".to_owned(),
12706            verified_head: None,
12707            verified_at: None,
12708            reviews: Vec::new(),
12709            e2e: vec![CommandOutcome {
12710                command: "test".to_owned(),
12711                code: Some(if e2e_ok { 0 } else { 1 }),
12712                output_tail: String::new(),
12713                duration_ms: 0,
12714                resource_blocked: false,
12715            }],
12716            verify_retried: false,
12717            e2e_deferred: false,
12718            e2e_defer_reason: None,
12719            fix: None,
12720            blocking,
12721            answered,
12722            expected,
12723            clean,
12724            progressed,
12725            vote_split: false,
12726            reconsideration: Vec::new(),
12727            verdict: None,
12728        }
12729    }
12730
12731    #[test]
12732    fn review_conclusion_is_none_when_nothing_has_run() {
12733        assert_eq!(review_conclusion(&[], 3), None);
12734    }
12735
12736    #[test]
12737    fn review_conclusion_is_none_while_rounds_remain() {
12738        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12739        assert_eq!(review_conclusion(&rounds, 3), None);
12740    }
12741
12742    #[test]
12743    fn review_conclusion_is_gating_once_a_round_is_clean() {
12744        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12745        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12746    }
12747
12748    #[test]
12749    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12750        let rounds = vec![
12751            review_round(false, 1, 2, 2, true, true),
12752            review_round(false, 1, 2, 2, true, true),
12753        ];
12754        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12755    }
12756
12757    #[test]
12758    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12759        let rounds = vec![
12760            review_round(false, 1, 2, 2, true, true),
12761            review_round(false, 1, 2, 2, true, false),
12762        ];
12763        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12764    }
12765
12766    #[test]
12767    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12768        // Magi never got a command to run against this round's own head — a
12769        // resource-blocked attempt, not a red one — so this must never
12770        // settle on `Blocked` the way a genuine e2e failure would. `None`
12771        // here is what tells `Runner::review_loop` to retry the check
12772        // itself rather than trust this cheap recomputation with a verdict
12773        // it cannot actually produce.
12774        let mut blocked = review_round(false, 1, 2, 2, true, false);
12775        blocked.e2e[0].resource_blocked = true;
12776        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12777        assert_eq!(review_conclusion(&rounds, 2), None);
12778    }
12779
12780    #[test]
12781    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12782        // Missing input, not a verified tree — never a hand-off candidate.
12783        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12784        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12785    }
12786
12787    #[test]
12788    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12789        let rounds = vec![
12790            review_round(false, 1, 2, 2, false, true),
12791            review_round(false, 1, 2, 2, false, true),
12792        ];
12793        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12794    }
12795
12796    fn secs(n: u64) -> Duration {
12797        Duration::from_secs(n)
12798    }
12799
12800    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12801    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12802    #[tokio::test]
12803    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12804        ask_test_home();
12805        let tmp = tempfile::tempdir().unwrap();
12806        let repo = tmp.path().join("repo");
12807        std::fs::create_dir_all(&repo).unwrap();
12808        init_repo(&repo);
12809        let g = |args: &[&str]| {
12810            let out = std::process::Command::new("git")
12811                .args(args)
12812                .current_dir(&repo)
12813                .quiet()
12814                .output()
12815                .unwrap();
12816            assert!(
12817                out.status.success(),
12818                "{}",
12819                String::from_utf8_lossy(&out.stderr)
12820            );
12821        };
12822        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12823        g(&["checkout", "-b", "historical"]);
12824        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12825        g(&["add", "."]);
12826        g(&["commit", "-m", "old alternative"]);
12827        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12828        g(&["checkout", "-b", "magi/task-selected", &base]);
12829        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12830        g(&["add", "."]);
12831        g(&["commit", "-m", "measurement infrastructure"]);
12832        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12833        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12834        g(&["add", "."]);
12835        g(&["commit", "-m", "dependent implementation"]);
12836        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12837        g(&["checkout", "main"]);
12838        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12839        g(&["add", "."]);
12840        g(&["commit", "-m", "unrelated base change"]);
12841        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12842        let origin = tmp.path().join("origin.git");
12843        g(&[
12844            "clone",
12845            "--bare",
12846            repo.to_str().unwrap(),
12847            origin.to_str().unwrap(),
12848        ]);
12849        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12850        g(&["fetch", "origin"]);
12851        let selection = crate::pr_recovery::Selection {
12852            url: "https://example.test/project/pull/1".into(),
12853            branch: "magi/task-selected".into(),
12854            head: head.clone(),
12855            source_head: None,
12856            source_run: None,
12857            source_proof: None,
12858            disposition: Some(crate::pr_recovery::Disposition::Review),
12859            instruction: "preserve measured behavior".into(),
12860            approved_at: jiff::Timestamp::now(),
12861        };
12862        let mut template = runner_at(RunStatus::Reviewing);
12863        template.state.repo = repo.clone();
12864        template.state.base_commit = current.clone();
12865        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12866        template.state.origin = Some(crate::run::Origin::queue("selected"));
12867        template.state.pr_head = Some(selection.branch.clone());
12868        template.state.pr_recovery = Some(selection.clone());
12869        let mut runner =
12870            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12871                .await
12872                .unwrap();
12873        let wt = runner.state.winner().unwrap().worktree.clone();
12874        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12875        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12876        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12877        assert!(runner.state.seeds.is_empty());
12878        assert_eq!(runner.state.instruction, "preserve measured behavior");
12879        runner.sync_to_base().await.unwrap();
12880        assert_eq!(
12881            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12882            "prerequisite"
12883        );
12884        assert_eq!(
12885            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12886            "measured implementation"
12887        );
12888        assert_eq!(
12889            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12890            "current main change"
12891        );
12892        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12893        assert_eq!(
12894            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12895            head
12896        );
12897
12898        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12899        let mut partial = runner_at(RunStatus::Reviewing);
12900        partial.state.repo = repo.clone();
12901        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12902        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12903            run: Some("exit 1".into()),
12904            ..Default::default()
12905        }];
12906        let selection = runner.state.pr_recovery.clone().unwrap();
12907        partial.state.pr_recovery = Some(selection.clone());
12908        partial.state.pr_head = Some(selection.branch.clone());
12909        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12910        let id = partial.state.id.clone();
12911        let error = match Runner::open_selected(
12912            &repo,
12913            partial.state,
12914            partial.roles,
12915            1,
12916            runner.state.base_commit.clone(),
12917            selection,
12918        )
12919        .await
12920        {
12921            Ok(_) => panic!("setup must fail"),
12922            Err(e) => e,
12923        };
12924        assert_eq!(
12925            error
12926                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12927                .unwrap()
12928                .run,
12929            id
12930        );
12931        let failed = RunState::load(&id).unwrap();
12932        assert_eq!(failed.status, RunStatus::Blocked);
12933        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12934        assert!(Runner::resume(&id).is_err());
12935        runner.state.status = RunStatus::Ready;
12936        runner.state.driver_exited = true;
12937        runner.state.save().unwrap();
12938        let mut retry = runner_at(RunStatus::Reviewing);
12939        retry.state.repo = repo.clone();
12940        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12941        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12942        reselected.approved_at = jiff::Timestamp::now();
12943        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12944        retry.state.pr_head = Some(reselected.branch.clone());
12945        retry.state.pr_recovery = Some(reselected.clone());
12946        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12947            .await
12948            .unwrap();
12949        let restarted = Runner::open_selected(
12950            &repo,
12951            retry.state,
12952            retry.roles,
12953            1,
12954            runner.state.base_commit.clone(),
12955            reselected,
12956        )
12957        .await
12958        .unwrap();
12959        assert_eq!(
12960            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12961                .await
12962                .unwrap(),
12963            head
12964        );
12965        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12966        assert_eq!(
12967            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12968            head
12969        );
12970    }
12971
12972    #[tokio::test]
12973    #[cfg(unix)]
12974    async fn fallback_and_interrupted_restoration_require_coordination() {
12975        use std::os::fd::AsRawFd as _;
12976        ask_test_home();
12977        let tmp = tempfile::tempdir().unwrap();
12978        let mut runner = registered_runner(tmp.path()).await;
12979        runner.state.config.graph.timeout_resource_wait = 1;
12980        let winner = runner.state.candidates[0].clone();
12981        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12982        let leftover = winner.worktree.join("leftover");
12983        std::fs::write(&leftover, "untrusted edits").unwrap();
12984        let next = tmp.path().join("next-fixer");
12985        let mut failed = spec("first");
12986        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12987        let mut successor = spec("second");
12988        successor.command = vec![
12989            "sh".into(),
12990            "-c".into(),
12991            format!("touch '{}'; echo done", next.display()),
12992        ];
12993        let attempts = vec![
12994            (failed.clone(), "first".into()),
12995            (successor.clone(), "second".into()),
12996        ];
12997        let lock_path = runner.state.repo.join(".git/updater.lock");
12998        let external = std::fs::File::create(&lock_path).unwrap();
12999        assert_eq!(
13000            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13001            0
13002        );
13003        let artifacts = tmp.path().join("artifacts");
13004        let build = |spec, seat| SeatJob {
13005            spec,
13006            seat,
13007            cwd: winner.worktree.clone(),
13008            prompt: "fixture".into(),
13009            timeout: Duration::from_secs(10),
13010            allow_write: true,
13011            sessions: false,
13012            artifacts: artifacts.clone(),
13013            stem: "fix".into(),
13014            handover: None,
13015        };
13016        let (_, _, out) = runner
13017            .ask_fixer_among(
13018                attempts.clone(),
13019                "graft",
13020                None,
13021                Some((&winner.worktree, &head)),
13022                build,
13023            )
13024            .await;
13025        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13026        assert!(leftover.exists(), "contention must prevent clean");
13027        assert!(!next.exists());
13028        assert!(runner.state.handovers.is_empty());
13029        // Early interrupted-repair rollback uses the same protected route.
13030        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13031            from_run: "earlier".into(),
13032            generation: 1,
13033            mode: crate::recovery::Mode::VerifyRepair,
13034            reasons: Vec::new(),
13035            reviewed_head: None,
13036            rounds: Vec::new(),
13037            open_findings: Vec::new(),
13038            evidence: None,
13039            criteria_inherited: true,
13040            repair: crate::recovery::Repair::Started {
13041                before: head.clone(),
13042            },
13043        });
13044        assert!(
13045            runner
13046                .recovery_repair(&winner)
13047                .await
13048                .unwrap_err()
13049                .is::<crate::repo_lock::ResourceBlocked>()
13050        );
13051        assert!(matches!(
13052            runner.state.recovery.as_ref().unwrap().repair,
13053            crate::recovery::Repair::Started { .. }
13054        ));
13055        assert!(leftover.exists());
13056        drop(external);
13057        runner.recovery_repair(&winner).await.unwrap();
13058        assert!(!leftover.exists());
13059        assert!(matches!(
13060            runner.state.recovery.as_ref().unwrap().repair,
13061            crate::recovery::Repair::Done { .. }
13062        ));
13063        // A hard lock-open failure also leaves HEAD and untracked work intact.
13064        std::fs::remove_file(&lock_path).unwrap();
13065        std::fs::create_dir(&lock_path).unwrap();
13066        std::fs::write(&leftover, "untrusted edits").unwrap();
13067        let (_, _, out) = runner
13068            .ask_fixer_among(
13069                attempts,
13070                "graft",
13071                None,
13072                Some((&winner.worktree, &head)),
13073                build,
13074            )
13075            .await;
13076        assert!(matches!(out, AgentOutcome::Failed(_)));
13077        assert!(leftover.exists());
13078        assert!(!next.exists());
13079        assert_eq!(
13080            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13081            head
13082        );
13083    }
13084
13085    #[cfg(unix)]
13086    async fn registered_runner(root: &Path) -> Runner {
13087        ask_test_home();
13088        let repo = root.join("repo");
13089        std::fs::create_dir_all(&repo).unwrap();
13090        init_repo(&repo);
13091        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13092        let mut runner = runner_at(RunStatus::Prep);
13093        runner.state.repo = repo.clone();
13094        runner.state.config.coordination = crate::config::test_coordination();
13095        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13096        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13097        runner.state.config.graph.advise = false;
13098        runner.roles.implementers = vec![spec("implement")];
13099        runner.roles.judges = vec![spec("judge")];
13100        runner.prep().await.unwrap();
13101        runner
13102    }
13103
13104    #[tokio::test]
13105    #[cfg(unix)]
13106    async fn parked_implementer_persists_completed_work() {
13107        for registered in [false, true] {
13108            let tmp = tempfile::tempdir().unwrap();
13109            let mut runner = registered_runner(tmp.path()).await;
13110            if !registered {
13111                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13112            }
13113            let marker = tmp.path().join("started");
13114            let release = tmp.path().join("release");
13115            let mut implementer = spec("implement");
13116            implementer.command = vec![
13117                "sh".into(),
13118                "-c".into(),
13119                format!(
13120                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13121                    marker.display(),
13122                    release.display()
13123                ),
13124            ];
13125            runner.roles.implementers = vec![implementer.clone()];
13126            runner.state.config.agents = vec![implementer];
13127            let pause = runner.pause.clone();
13128            let (result, ()) = tokio::join!(runner.implement(), async {
13129                tokio::time::timeout(Duration::from_secs(10), async {
13130                    while !marker.exists() {
13131                        tokio::time::sleep(Duration::from_millis(10)).await;
13132                    }
13133                })
13134                .await
13135                .unwrap();
13136                pause.park();
13137                std::fs::write(&release, "go").unwrap();
13138            });
13139            result.unwrap();
13140            let candidate = &runner.state.candidates[0];
13141            assert!(candidate.commits > 0, "{candidate:?}");
13142            assert!(candidate.summary.contains("saved-result"));
13143            assert_eq!(candidate.files, 1);
13144            let before = candidate.commits;
13145            runner.implement().await.unwrap();
13146            assert_eq!(runner.state.candidates[0].commits, before);
13147        }
13148    }
13149
13150    #[tokio::test]
13151    #[cfg(unix)]
13152    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13153        let tmp = tempfile::tempdir().unwrap();
13154        let mut runner = registered_runner(tmp.path()).await;
13155        let winner = runner.state.candidates[0].clone();
13156        let cache = tmp.path().join("cache");
13157        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13158        runner.state.config.graph.timeout_resource_wait = 0;
13159        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13160        let crate::cache::AcquireOutcome::Acquired(held) =
13161            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13162        else {
13163            panic!("cache must be free")
13164        };
13165        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13166            from_run: "earlier".into(),
13167            generation: 1,
13168            mode: crate::recovery::Mode::VerifyRepair,
13169            reasons: Vec::new(),
13170            reviewed_head: None,
13171            rounds: Vec::new(),
13172            open_findings: Vec::new(),
13173            evidence: None,
13174            criteria_inherited: true,
13175            repair: crate::recovery::Repair::NotStarted,
13176        });
13177        runner.recovery_repair(&winner).await.unwrap();
13178        assert!(matches!(
13179            runner.state.recovery.as_ref().unwrap().repair,
13180            crate::recovery::Repair::WaitingForRestore { .. }
13181        ));
13182        drop(held);
13183        runner.state.resource_block = None;
13184        runner.state.config.graph.timeout_resource_wait = 1;
13185        runner.recovery_repair(&winner).await.unwrap();
13186        assert!(matches!(
13187            runner.state.recovery.as_ref().unwrap().repair,
13188            crate::recovery::Repair::Done { .. }
13189        ));
13190        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13191    }
13192
13193    #[tokio::test]
13194    #[cfg(unix)]
13195    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13196        use std::os::fd::AsRawFd as _;
13197        for graft in [false, true] {
13198            let tmp = tempfile::tempdir().unwrap();
13199            let mut runner = registered_runner(tmp.path()).await;
13200            runner.state.config.graph.timeout_resource_wait = 1;
13201            let winner = runner.state.candidates[0].clone();
13202            let marker = tmp.path().join("fixer-started");
13203            let mut fixer = spec("fixer");
13204            fixer.command = vec![
13205                "sh".into(),
13206                "-c".into(),
13207                format!("echo started >> '{}'; echo done", marker.display()),
13208            ];
13209            runner.roles.fixer = Some(vec![fixer]);
13210            if graft {
13211                let mut loser = winner.clone();
13212                loser.label = 'B';
13213                loser.index = 1;
13214                runner.state.candidates.push(loser);
13215                runner.state.config.graph.graft_runner_up = true;
13216                runner.state.tally = Some(Tally {
13217                    first_choice: BTreeMap::new(),
13218                    borda: BTreeMap::new(),
13219                    winner: winner.label,
13220                    rankings: 1,
13221                    unanimous_initial: true,
13222                    deliberated: false,
13223                    changed_votes: 0,
13224                    unanimous_final: true,
13225                    tie_break: None,
13226                    judges: 1,
13227                    present: 1,
13228                    quorum: 1,
13229                    met_quorum: true,
13230                    uncontested: None,
13231                });
13232                runner.state.judgements.push(crate::run::Judgement {
13233                    judge: 1,
13234                    seat: "judge-1".into(),
13235                    agent: "judge".into(),
13236                    ranking: vec![winner.label, 'B'],
13237                    reasons: BTreeMap::new(),
13238                    confidence: None,
13239                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13240                    order: vec![0, 1],
13241                    failed: None,
13242                    duration_ms: 0,
13243                });
13244            } else {
13245                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13246                    from_run: "earlier".into(),
13247                    generation: 1,
13248                    mode: crate::recovery::Mode::VerifyRepair,
13249                    reasons: Vec::new(),
13250                    reviewed_head: None,
13251                    rounds: Vec::new(),
13252                    open_findings: Vec::new(),
13253                    evidence: None,
13254                    criteria_inherited: true,
13255                    repair: crate::recovery::Repair::NotStarted,
13256                });
13257            }
13258            let reader = crate::repo_lock::SeatLease::for_checkout(
13259                &winner.worktree,
13260                false,
13261                crate::config::test_coordination().as_ref(),
13262            )
13263            .await
13264            .unwrap();
13265            let pause = runner.pause.clone();
13266            let (result, ()) = tokio::join!(
13267                async {
13268                    if graft {
13269                        runner.graft().await
13270                    } else {
13271                        runner.recovery_repair(&winner).await
13272                    }
13273                },
13274                async {
13275                    tokio::time::sleep(Duration::from_millis(100)).await;
13276                    pause.park();
13277                }
13278            );
13279            result.unwrap();
13280            assert!(!marker.exists(), "a blocked seat must not spawn");
13281            // Exercise the persisted state, as a binary replacement/resume does.
13282            runner.state = RunState::load(&runner.state.id).unwrap();
13283            if graft {
13284                assert_eq!(
13285                    runner.state.graft.as_ref().unwrap().outcome,
13286                    GraftOutcome::WaitingForRestore
13287                );
13288            } else {
13289                assert!(matches!(
13290                    runner.state.recovery.as_ref().unwrap().repair,
13291                    crate::recovery::Repair::WaitingForRestore { .. }
13292                ));
13293            }
13294            drop(reader);
13295            runner.pause = Pause::new();
13296            runner.state.resource_block = None;
13297            // Deferred restoration must still respect an external lock owner.
13298            let external =
13299                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13300            assert_eq!(
13301                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13302                0
13303            );
13304            let leftover = winner.worktree.join("leftover");
13305            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13306            let result = if graft {
13307                runner.graft().await
13308            } else {
13309                runner.recovery_repair(&winner).await
13310            };
13311            assert!(
13312                result
13313                    .unwrap_err()
13314                    .is::<crate::repo_lock::ResourceBlocked>()
13315            );
13316            assert!(leftover.exists());
13317            assert!(!marker.exists());
13318            drop(external);
13319            runner.state.resource_block = None;
13320            let result = if graft {
13321                runner.graft().await
13322            } else {
13323                runner.recovery_repair(&winner).await
13324            };
13325            result.unwrap();
13326            assert!(!leftover.exists(), "restore must precede retry");
13327            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13328            if graft {
13329                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13330            } else {
13331                assert!(matches!(
13332                    runner.state.recovery.as_ref().unwrap().repair,
13333                    crate::recovery::Repair::Done { .. }
13334                ));
13335            }
13336        }
13337    }
13338
13339    #[tokio::test]
13340    #[cfg(unix)]
13341    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13342        use std::os::fd::AsRawFd as _;
13343        let tmp = tempfile::tempdir().unwrap();
13344        let mut runner = registered_runner(tmp.path()).await;
13345        runner.state.config.graph.timeout_resource_wait = 1;
13346        let winner = runner.state.candidates[0].clone();
13347        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13348        runner.state.tally = Some(Tally {
13349            first_choice: BTreeMap::new(),
13350            borda: BTreeMap::new(),
13351            winner: winner.label,
13352            rankings: 0,
13353            unanimous_initial: false,
13354            deliberated: false,
13355            changed_votes: 0,
13356            unanimous_final: false,
13357            tie_break: None,
13358            judges: 0,
13359            present: 0,
13360            quorum: 0,
13361            met_quorum: true,
13362            uncontested: Some("fixture".into()),
13363        });
13364        runner.state.graft = Some(GraftRecord {
13365            pre: pre.clone(),
13366            post: None,
13367            lines: None,
13368            limit: None,
13369            agent: None,
13370            outcome: GraftOutcome::Running,
13371            requested: Vec::new(),
13372            adoption: None,
13373            adoption_error: None,
13374        });
13375        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13376        git::commit_all(&winner.worktree, "unfinished graft")
13377            .await
13378            .unwrap();
13379        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13380        let leftover = winner.worktree.join("leftover");
13381        std::fs::write(&leftover, "untrusted edits").unwrap();
13382        let path = runner.state.repo.join(".git/updater.lock");
13383        let external = std::fs::File::create(&path).unwrap();
13384        assert_eq!(
13385            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13386            0
13387        );
13388        assert!(
13389            runner
13390                .graft()
13391                .await
13392                .unwrap_err()
13393                .is::<crate::repo_lock::ResourceBlocked>()
13394        );
13395        assert!(matches!(
13396            runner.state.graft.as_ref().unwrap().outcome,
13397            GraftOutcome::Running
13398        ));
13399        assert_eq!(
13400            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13401            unfinished
13402        );
13403        assert!(leftover.exists());
13404        drop(external);
13405        std::fs::remove_file(&path).unwrap();
13406        std::fs::create_dir(&path).unwrap();
13407        assert!(runner.graft().await.is_err());
13408        assert!(matches!(
13409            runner.state.graft.as_ref().unwrap().outcome,
13410            GraftOutcome::Running
13411        ));
13412        assert_eq!(
13413            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13414            unfinished
13415        );
13416        assert!(leftover.exists());
13417        std::fs::remove_dir(&path).unwrap();
13418        runner.graft().await.unwrap();
13419        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13420        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13421        assert!(!leftover.exists());
13422    }
13423
13424    #[tokio::test]
13425    #[cfg(unix)]
13426    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13427        use std::os::fd::AsRawFd as _;
13428        let tmp = tempfile::tempdir().unwrap();
13429        let mut runner = registered_runner(tmp.path()).await;
13430        runner.state.prep_complete = Some(false);
13431        runner.state.config.graph.timeout_resource_wait = 1;
13432        runner.state.save().unwrap();
13433        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13434        assert_eq!(
13435            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13436            0
13437        );
13438        runner.execute().await.unwrap();
13439        let disk = RunState::load(&runner.state.id).unwrap();
13440        assert!(disk.driver_exited);
13441        assert_eq!(disk.driver_pid, None);
13442        assert_eq!(runner.state.status, RunStatus::Blocked);
13443        assert_eq!(
13444            runner.state.resource_block.as_ref().unwrap().kind,
13445            "repository coordination"
13446        );
13447        assert!(runner.state.prep_failure.is_none());
13448        assert!(runner.state.seats.is_empty());
13449        runner.pause.park();
13450        // Include headroom for Git subprocesses under parallel test load;
13451        // the repository contention deadline itself remains one second.
13452        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13453            .await
13454            .unwrap()
13455            .unwrap();
13456        assert!(runner.state.parked);
13457        assert!(runner.state.prep_failure.is_none());
13458        let disk = RunState::load(&runner.state.id).unwrap();
13459        assert!(disk.driver_exited);
13460        assert_eq!(disk.driver_pid, None);
13461    }
13462
13463    #[tokio::test]
13464    #[cfg(unix)]
13465    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13466        use std::os::fd::AsRawFd as _;
13467        let tmp = tempfile::tempdir().unwrap();
13468        let mut runner = registered_runner(tmp.path()).await;
13469        runner.state.prep_complete = Some(false);
13470        runner.state.config.graph.timeout_resource_wait = 30;
13471        runner.state.save().unwrap();
13472        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13473        assert_eq!(
13474            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13475            0
13476        );
13477        assert!(
13478            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13479                .await
13480                .is_err()
13481        );
13482        let disk = RunState::load(&runner.state.id).unwrap();
13483        assert!(disk.driver_exited);
13484        assert_eq!(disk.driver_pid, None);
13485        assert!(disk.status.resumable());
13486        assert!(runner.state.driver_exited);
13487        assert_eq!(runner.state.driver_pid, None);
13488        runner.state.save().unwrap();
13489        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13490    }
13491
13492    #[tokio::test]
13493    #[cfg(unix)]
13494    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13495        let tmp = tempfile::tempdir().unwrap();
13496        let mut runner = registered_runner(tmp.path()).await;
13497        let winner = runner.state.candidates[0].clone();
13498        let cache = tmp.path().join("cache");
13499        let started = tmp.path().join("pre-gate-started");
13500        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13501        runner.state.config.verify.gate =
13502            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13503        let home = crate::run::home();
13504        let owner =
13505            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13506        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13507            crate::cache::AcquireOutcome::Acquired(g) => g,
13508            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13509        };
13510        let mut gate = Box::pin(runner.run_gate(&winner));
13511        assert!(
13512            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13513                .await
13514                .is_err()
13515        );
13516        assert!(!started.exists(), "pre-gate must wait for the cache too");
13517        // Simulate the wave that already owns the cache entering invoke.
13518        // It must obtain the editor reservation and finish without waiting
13519        // for the gate that is queued behind its cache.
13520        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13521        let editor = crate::repo_lock::SeatLease::checkout_with(
13522            &winner.worktree,
13523            true,
13524            crate::config::test_coordination().as_ref(),
13525            &wait,
13526        )
13527        .await
13528        .unwrap();
13529        drop(editor);
13530        guard.release();
13531        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13532            .await
13533            .unwrap()
13534            .unwrap();
13535        drop(gate);
13536        assert!(outcomes.iter().all(CommandOutcome::ok));
13537        assert!(started.exists());
13538    }
13539
13540    #[tokio::test]
13541    #[cfg(unix)]
13542    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13543        use std::os::fd::AsRawFd as _;
13544
13545        ask_test_home();
13546        let tmp = tempfile::tempdir().unwrap();
13547        let repo = tmp.path().join("repo");
13548        std::fs::create_dir_all(&repo).unwrap();
13549        init_repo(&repo);
13550        let common = repo.join(".git");
13551        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13552        let mut runner = runner_at(RunStatus::Prep);
13553        runner.state.repo = repo.clone();
13554        runner.state.config.coordination = crate::config::test_coordination();
13555        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13556        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13557        runner.state.config.graph.advise = false;
13558        runner.roles.implementers = vec![spec("implement")];
13559        runner.roles.judges = vec![spec("judge")];
13560        runner.prep().await.unwrap();
13561        let winner = runner.state.candidates[0].clone();
13562        // Emulate edits already left by a completed seat. The fixture CLI
13563        // only reports completion; it performs no repository operations.
13564        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13565        let completed = tmp.path().join("seat-completed");
13566        runner.roles.implementers[0].command = vec![
13567            "sh".into(),
13568            "-c".into(),
13569            format!("touch '{}'; echo completed", completed.display()),
13570        ];
13571        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13572        assert_eq!(
13573            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13574            0
13575        );
13576        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13577        let mut implement = Box::pin(runner.implement());
13578        tokio::select! {
13579            result = &mut implement => panic!("rescue must wait: {result:?}"),
13580            result = tokio::time::timeout(Duration::from_secs(10), async {
13581                while !completed.exists() {
13582                    tokio::time::sleep(Duration::from_millis(20)).await;
13583                }
13584            }) => result.unwrap(),
13585        }
13586        assert!(
13587            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13588                .await
13589                .is_err()
13590        );
13591        assert_eq!(
13592            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13593            before
13594        );
13595        assert_eq!(
13596            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13597            0
13598        );
13599        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13600            .await
13601            .unwrap()
13602            .unwrap();
13603        drop(implement);
13604        assert_ne!(
13605            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13606            before
13607        );
13608        assert!(git::is_clean(&winner.worktree).await.unwrap());
13609
13610        let started = tmp.path().join("pre-gate-started");
13611        let proceed = tmp.path().join("pre-gate-proceed");
13612        let checked = tmp.path().join("gate-ran");
13613        runner.state.config.verify.pre_gate = vec![format!(
13614            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13615            started.display(),
13616            proceed.display()
13617        )];
13618        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13619        assert_eq!(
13620            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13621            0
13622        );
13623        let mut gate = Box::pin(runner.run_gate(&winner));
13624        assert!(
13625            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13626                .await
13627                .is_err()
13628        );
13629        assert!(!started.exists());
13630        assert!(!checked.exists());
13631        assert!(git::is_clean(&winner.worktree).await.unwrap());
13632        assert_eq!(
13633            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13634            0
13635        );
13636        tokio::select! {
13637            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13638            result = tokio::time::timeout(Duration::from_secs(10), async {
13639                while !started.exists() {
13640                    tokio::time::sleep(Duration::from_millis(20)).await;
13641                }
13642            }) => result.unwrap(),
13643        }
13644        assert_eq!(
13645            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13646            -1,
13647            "mechanical edits must hold the real lock"
13648        );
13649        std::fs::write(proceed, "ready").unwrap();
13650        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13651            .await
13652            .unwrap()
13653            .unwrap();
13654        drop(gate);
13655        assert!(outcomes.iter().all(CommandOutcome::ok));
13656        assert!(checked.exists());
13657        assert!(runner.state.pre_gate_commit.is_some());
13658        assert!(git::is_clean(&winner.worktree).await.unwrap());
13659        assert_eq!(
13660            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13661            0,
13662            "gate completion must release coordination"
13663        );
13664        drop(outsider);
13665        std::fs::remove_file(common.join("updater.lock")).unwrap();
13666        std::fs::create_dir(common.join("updater.lock")).unwrap();
13667        std::fs::remove_file(&started).unwrap();
13668        std::fs::remove_file(&checked).unwrap();
13669        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13670        let error = runner.run_gate(&winner).await.unwrap_err();
13671        assert!(
13672            error
13673                .to_string()
13674                .contains("acquire mandatory repository lock")
13675        );
13676        assert!(
13677            !started.exists(),
13678            "failed acquisition must prevent pre-gate"
13679        );
13680        assert!(!checked.exists(), "failed acquisition must prevent gate");
13681        assert!(git::is_clean(&winner.worktree).await.unwrap());
13682        assert_eq!(
13683            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13684            head
13685        );
13686    }
13687
13688    #[tokio::test]
13689    #[cfg(unix)]
13690    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13691        use std::os::fd::AsRawFd as _;
13692        use std::time::Duration;
13693
13694        ask_test_home();
13695        let tmp = tempfile::tempdir().unwrap();
13696        let repo = tmp.path().join("repo");
13697        std::fs::create_dir_all(&repo).unwrap();
13698        init_repo(&repo);
13699        let common = repo.join(".git");
13700        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13701        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13702        assert_eq!(
13703            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13704            0
13705        );
13706        let mut runner = runner_at(RunStatus::Prep);
13707        runner.state.repo = repo.clone();
13708        runner.state.config.coordination = crate::config::test_coordination();
13709        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13710        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13711        runner.state.config.graph.advise = true;
13712        runner.state.config.graph.advisors = 1;
13713        runner.roles.implementers = vec![spec("implement")];
13714        runner.roles.judges = vec![spec("judge")];
13715        let root = runner.state.worktree_root();
13716        let started = tmp.path().join("setup-started");
13717        let proceed = tmp.path().join("setup-proceed");
13718        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13719            run: Some(format!(
13720                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13721                started.display(),
13722                proceed.display()
13723            )),
13724            ..Default::default()
13725        }];
13726        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13727        let config = git::git(&repo, &["config", "--local", "--list"])
13728            .await
13729            .unwrap();
13730        let mut prep = Box::pin(runner.prep());
13731        assert!(
13732            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13733                .await
13734                .is_err()
13735        );
13736        assert!(
13737            !root.exists(),
13738            "contention must prevent even checkout creation"
13739        );
13740        assert!(!started.exists(), "contention must prevent setup");
13741        assert_eq!(
13742            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13743            branches
13744        );
13745        assert_eq!(
13746            git::git(&repo, &["config", "--local", "--list"])
13747                .await
13748                .unwrap(),
13749            config
13750        );
13751        drop(lock);
13752        tokio::select! {
13753            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13754            result = tokio::time::timeout(Duration::from_secs(10), async {
13755                while !started.exists() {
13756                    tokio::time::sleep(Duration::from_millis(20)).await;
13757                }
13758            }) => result.unwrap(),
13759        }
13760        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13761        assert_eq!(
13762            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13763            -1
13764        );
13765        assert!(
13766            tokio::time::timeout(
13767                Duration::from_millis(100),
13768                crate::repo_lock::SeatLease::for_checkout(
13769                    &repo,
13770                    true,
13771                    crate::config::test_coordination().as_ref()
13772                )
13773            )
13774            .await
13775            .is_err(),
13776            "preparation must also exclude local editors"
13777        );
13778        std::fs::write(proceed, "ready").unwrap();
13779        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13780            .await
13781            .unwrap()
13782            .unwrap();
13783        drop(prep);
13784        assert_eq!(runner.state.prep_complete, Some(true));
13785        assert!(root.join("judge-1").exists());
13786        assert!(root.join("advisor-1").exists());
13787        assert_eq!(
13788            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13789            0
13790        );
13791    }
13792
13793    #[tokio::test]
13794    #[cfg(unix)]
13795    async fn preparation_lock_failure_prevents_repository_changes() {
13796        ask_test_home();
13797        let tmp = tempfile::tempdir().unwrap();
13798        let repo = tmp.path().join("repo");
13799        std::fs::create_dir_all(&repo).unwrap();
13800        init_repo(&repo);
13801        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13802        // A directory at the required lock path makes opening the real lock
13803        // fail, rather than merely exercising an invalid registration.
13804        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13805        let mut runner = runner_at(RunStatus::Prep);
13806        runner.state.repo = repo.clone();
13807        runner.state.config.coordination = crate::config::test_coordination();
13808        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13809        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13810        runner.roles.implementers = vec![spec("implement")];
13811        let error = runner.prep().await.unwrap_err();
13812        assert!(
13813            error
13814                .to_string()
13815                .contains("acquire mandatory repository lock")
13816        );
13817        assert!(!runner.state.worktree_root().exists());
13818        assert!(runner.state.candidates.is_empty());
13819        assert!(!repo.join(".git/config.worktree").exists());
13820        assert_eq!(
13821            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13822            "* main"
13823        );
13824    }
13825
13826    #[tokio::test]
13827    async fn preparation_records_creation_intent_before_git_can_fail() {
13828        ask_test_home();
13829        let tmp = tempfile::tempdir().unwrap();
13830        let repo = tmp.path().join("repo");
13831        std::fs::create_dir_all(&repo).unwrap();
13832        init_repo(&repo);
13833        let mut runner = runner_at(RunStatus::Prep);
13834        runner.state.repo = repo.clone();
13835        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13836        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13837        runner.state.config.blind.commit_msg_hook = false;
13838        runner.roles.implementers = vec![spec("one")];
13839        let label = blind::assign_labels(1, runner.state.seed)[0];
13840        let branch = runner.state.branch_for(label);
13841        // A conflicting branch makes Git refuse creation before a candidate
13842        // can be registered. The durable intent must still identify it.
13843        git::git(&repo, &["branch", &branch]).await.unwrap();
13844        assert!(runner.prep().await.is_err());
13845        let saved = RunState::load(&runner.state.id).unwrap();
13846        assert!(saved.candidates.is_empty());
13847        let event = saved
13848            .events
13849            .iter()
13850            .find(|e| e.node == "prep-intent")
13851            .unwrap();
13852        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13853        assert_eq!(intent["branch"], branch);
13854        assert_eq!(intent["start"], runner.state.base_commit);
13855        assert_eq!(
13856            intent["worktree"],
13857            runner
13858                .state
13859                .worktree_root()
13860                .join(format!("cand-{label}"))
13861                .to_string_lossy()
13862                .as_ref()
13863        );
13864        assert_eq!(intent["picks"], serde_json::json!([]));
13865    }
13866
13867    #[tokio::test]
13868    async fn invalid_pinned_sources_create_no_partial_worktrees() {
13869        ask_test_home();
13870        let tmp = tempfile::tempdir().unwrap();
13871        let repo = tmp.path().join("repo");
13872        std::fs::create_dir_all(&repo).unwrap();
13873        init_repo(&repo);
13874        let mut runner = runner_at(RunStatus::Prep);
13875        runner.state.repo = repo.clone();
13876        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13877        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13878        runner.state.sources.pick_commits.push(refs::Input {
13879            token: "missing".into(),
13880            sha: "0".repeat(40),
13881            provenance: refs::Provenance::ExplicitCli,
13882        });
13883        assert!(runner.prep().await.is_err());
13884        assert!(runner.state.candidates.is_empty());
13885        assert!(!runner.state.worktree_root().exists());
13886    }
13887
13888    #[tokio::test]
13889    async fn prose_and_legacy_inference_never_import_into_candidates() {
13890        ask_test_home();
13891        let tmp = tempfile::tempdir().unwrap();
13892        let repo = tmp.path().join("repo");
13893        std::fs::create_dir_all(&repo).unwrap();
13894        init_repo(&repo);
13895        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13896        git::git(&repo, &["checkout", "-b", "magi/history/A"])
13897            .await
13898            .unwrap();
13899        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13900        git::git(&repo, &["add", "."]).await.unwrap();
13901        git::git(&repo, &["commit", "-m", "historical implementation"])
13902            .await
13903            .unwrap();
13904        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13905        git::git(&repo, &["checkout", "main"]).await.unwrap();
13906        for instruction in [
13907            format!("Example: magi/history/A at {source}"),
13908            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13909            format!("Never import magi/history/A or {source}"),
13910            format!("Conversation answer: I mentioned {source} only as background"),
13911        ] {
13912            let mut runner = runner_at(RunStatus::Prep);
13913            runner.state.repo = repo.clone();
13914            runner.state.base_commit = base.clone();
13915            runner.state.instruction = instruction.clone();
13916            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13917            runner.state.config.blind.commit_msg_hook = false;
13918            runner.state.seeds = vec![refs::Seed {
13919                token: "magi/history/A".into(),
13920                sha: source.clone(),
13921                kind: refs::SeedKind::Unmerged,
13922                branch: true,
13923                detail: String::new(),
13924            }];
13925            runner.roles.implementers = vec![spec("one")];
13926            assert_eq!(seeded_instruction(&runner.state), instruction);
13927            // Model an old record with inferred seeds but no structured input field.
13928            let mut legacy = serde_json::to_value(&runner.state).unwrap();
13929            legacy.as_object_mut().unwrap().remove("sources");
13930            runner.state = serde_json::from_value(legacy).unwrap();
13931            assert_eq!(runner.state.sources, refs::Sources::default());
13932            runner.state.save().unwrap();
13933            runner.state = RunState::load(&runner.state.id).unwrap();
13934            runner.prep().await.unwrap();
13935            for candidate in &runner.state.candidates {
13936                assert_eq!(
13937                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13938                    base
13939                );
13940                assert!(!candidate.worktree.join("unwanted").exists());
13941            }
13942        }
13943    }
13944
13945    #[tokio::test]
13946    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13947        ask_test_home();
13948        let tmp = tempfile::tempdir().unwrap();
13949        let repo = tmp.path().join("repo");
13950        std::fs::create_dir_all(&repo).unwrap();
13951        init_repo(&repo);
13952        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13953        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13954        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13955        git::git(&repo, &["add", "."]).await.unwrap();
13956        git::git(&repo, &["commit", "-m", "required seed"])
13957            .await
13958            .unwrap();
13959        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13960        git::git(&repo, &["checkout", "main"]).await.unwrap();
13961        let mut runner = runner_at(RunStatus::Prep);
13962        runner.state.repo = repo.clone();
13963        runner.state.base_commit = base.clone();
13964        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13965        runner.state.config.graph.advise = false;
13966        runner.state.config.blind.commit_msg_hook = false;
13967        runner.state.sources =
13968            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13969                .await
13970                .unwrap();
13971        runner.roles.implementers = vec![spec("one"), spec("two")];
13972        runner.roles.judges = vec![spec("judge")];
13973        let label = blind::assign_labels(2, runner.state.seed)[0];
13974        let branch = runner.state.branch_for(label);
13975        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13976        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13977            .await
13978            .unwrap();
13979        runner.state.candidates.push(Candidate {
13980            index: 0,
13981            label,
13982            agent: "one".into(),
13983            branch,
13984            worktree,
13985            summary: String::new(),
13986            stat: String::new(),
13987            files: 0,
13988            commits: 0,
13989            empty: false,
13990            failed: None,
13991            verified_noop: None,
13992            duration_ms: 0,
13993            folded: false,
13994        });
13995        runner.state.save().unwrap();
13996        // Reload the checkpoint immediately before its first cherry-pick.
13997        runner.state = RunState::load(&runner.state.id).unwrap();
13998        assert_eq!(runner.state.prep_complete, Some(false));
13999        runner.prep().await.unwrap();
14000        assert_eq!(runner.state.prep_complete, Some(true));
14001        assert_eq!(runner.state.candidates.len(), 2);
14002        for candidate in &runner.state.candidates {
14003            assert_eq!(
14004                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14005                git::tree_of(&repo, &seed).await.unwrap()
14006            );
14007            assert_eq!(
14008                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14009                "required commit"
14010            );
14011        }
14012        assert!(runner.state.worktree_root().join("judge-1").exists());
14013        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14014        runner.prep().await.unwrap();
14015        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14016
14017        runner.state.prep_complete = Some(false);
14018        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14019            run: Some(format!(
14020                "test -f '{}'",
14021                tmp.path().join("dependency-ready").display()
14022            )),
14023            ..Default::default()
14024        }];
14025        assert!(runner.prep().await.is_err());
14026        assert!(runner.state.prep_failure.is_some());
14027        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14028        runner.state = RunState::load(&runner.state.id).unwrap();
14029        assert!(
14030            runner.prep().await.is_err(),
14031            "correcting the cause alone must not auto-retry"
14032        );
14033        let failed_at = runner.state.events.last().unwrap().at;
14034        assert!(
14035            runner.retry_preparation(failed_at).is_err(),
14036            "old approvals cannot unlock the hold"
14037        );
14038        let mut task = crate::queue::Task::new(
14039            "retry".into(),
14040            "retry setup".into(),
14041            repo.clone(),
14042            crate::queue::Source::Human,
14043        );
14044        task.start(runner.state.id.clone());
14045        task.hold_machine(Some("setup failed".into()));
14046        task.release_operator("cli", None);
14047        runner
14048            .retry_preparation(task.resumption.as_ref().unwrap().at)
14049            .unwrap();
14050        runner.prep().await.unwrap();
14051        assert!(runner.state.prep_failure.is_none());
14052        assert_eq!(runner.state.prep_complete, Some(true));
14053        assert_eq!(runner.state.candidates.len(), 2);
14054        assert!(runner.state.worktree_root().join("judge-1").exists());
14055        for candidate in &runner.state.candidates {
14056            assert_eq!(
14057                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14058                git::tree_of(&repo, &seed).await.unwrap()
14059            );
14060        }
14061    }
14062
14063    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14064        let mut heads = Vec::new();
14065        for candidate in candidates {
14066            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14067        }
14068        heads
14069    }
14070
14071    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14072        ask_test_home();
14073        let tmp = tempfile::tempdir().unwrap();
14074        let repo = tmp.path().join("repo");
14075        std::fs::create_dir(&repo).unwrap();
14076        init_repo(&repo);
14077        let mut runner = runner_at(RunStatus::Reviewing);
14078        runner.state.repo = repo.clone();
14079        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14080        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14081        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14082            let branch = format!("candidate-{label}");
14083            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14084            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14085                .await
14086                .unwrap();
14087            if label == 'B' {
14088                std::fs::write(worktree.join("change"), "winner").unwrap();
14089                git::rescue_commit(&worktree, "candidate change")
14090                    .await
14091                    .unwrap();
14092            }
14093            runner.state.candidates.push(Candidate {
14094                index,
14095                label,
14096                agent: "alpha".into(),
14097                branch,
14098                worktree,
14099                summary: "completed reply".into(),
14100                stat: String::new(),
14101                files: usize::from(label == 'B'),
14102                commits: usize::from(label == 'B'),
14103                empty: label == 'A',
14104                failed: None,
14105                verified_noop: None,
14106                duration_ms: 1,
14107                folded: false,
14108            });
14109        }
14110        runner.state.judge_skipped = true;
14111        runner.state.tally = Some(Tally {
14112            first_choice: BTreeMap::new(),
14113            borda: BTreeMap::new(),
14114            winner: 'B',
14115            rankings: 0,
14116            unanimous_initial: false,
14117            deliberated: false,
14118            changed_votes: 0,
14119            unanimous_final: false,
14120            tie_break: None,
14121            judges: 0,
14122            present: 0,
14123            quorum: 0,
14124            met_quorum: true,
14125            uncontested: Some("only B produced a change".into()),
14126        });
14127        runner.state.save().unwrap();
14128        (tmp, runner)
14129    }
14130
14131    #[tokio::test]
14132    #[cfg(unix)]
14133    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14134        for (solo, failed_checkpoint) in
14135            [(true, false), (false, false), (true, true), (false, true)]
14136        {
14137            let (_tmp, mut runner) = preservation_resume_fixture().await;
14138            if solo {
14139                runner.state.candidates.retain(|c| c.label == 'A');
14140            }
14141            runner.state.candidates[0].empty = false;
14142            runner.state.candidates[0].summary.clear();
14143            runner.state.tally = None;
14144            runner.state.judge_skipped = false;
14145            runner.state.status = RunStatus::Implementing;
14146            runner.state.config.disk.min_free_bytes = u64::MAX;
14147            let mut implementer = spec("alpha");
14148            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14149            runner.roles.implementers = vec![implementer];
14150            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14151            if failed_checkpoint {
14152                std::fs::create_dir_all(&patch).unwrap();
14153            }
14154            runner.state.save().unwrap();
14155            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14156            runner.state = RunState::load(&runner.state.id).unwrap();
14157            assert_eq!(runner.state.status, RunStatus::Blocked);
14158            assert!(runner.state.interrupted_implementations.contains(&'A'));
14159            assert_eq!(runner.state.candidates[0].commits, 0);
14160            assert!(!runner.state.candidates[0].empty);
14161            assert!(runner.state.candidates[0].failed.is_none());
14162            if failed_checkpoint {
14163                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14164                std::fs::remove_dir(patch).unwrap();
14165            }
14166            runner.state.resource_block = None;
14167            runner.state.config.disk.min_free_bytes = 0;
14168            runner.roles.implementers[0].command = vec![
14169                "sh".into(),
14170                "-c".into(),
14171                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14172            ];
14173            runner.implement().await.unwrap();
14174            assert!(runner.state.interrupted_implementations.is_empty());
14175            assert!(runner.state.candidate_preservation.is_empty());
14176            assert_eq!(runner.state.candidates[0].commits, 1);
14177            assert!(!runner.state.candidates[0].empty);
14178            assert!(
14179                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14180                    .unwrap()
14181                    .contains("retried empty interrupted seat")
14182            );
14183            if !solo {
14184                assert_eq!(runner.state.candidates[1].commits, 1);
14185            }
14186        }
14187    }
14188
14189    #[tokio::test]
14190    #[cfg(unix)]
14191    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14192        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14193            let (_tmp, mut runner) = preservation_resume_fixture().await;
14194            runner.state.candidates.retain(|c| c.label == 'A');
14195            runner.state.candidates[0].empty = false;
14196            runner.state.candidates[0].summary.clear();
14197            runner.state.tally = None;
14198            runner.state.judge_skipped = false;
14199            runner.state.status = RunStatus::Implementing;
14200            // Force the real post-spawn watchdog, independently of available
14201            // disk space. Dirty/committed work predates its first measurement.
14202            runner.state.config.disk.min_free_bytes = u64::MAX;
14203            let wt = runner.state.candidates[0].worktree.clone();
14204            std::fs::write(
14205                wt.join("interrupted-source"),
14206                "retain implementation work\n",
14207            )
14208            .unwrap();
14209            if committed {
14210                git::rescue_commit(&wt, "implementation checkpoint")
14211                    .await
14212                    .unwrap();
14213            }
14214            let lock = git::git(
14215                &wt,
14216                &[
14217                    "rev-parse",
14218                    "--path-format=absolute",
14219                    "--git-path",
14220                    "index.lock",
14221                ],
14222            )
14223            .await
14224            .unwrap();
14225            if block_rescue {
14226                std::fs::write(&lock, "fixture-owned lock").unwrap();
14227            }
14228            let mut implementer = spec("alpha");
14229            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14230            runner.roles.implementers = vec![implementer];
14231            runner.state.save().unwrap();
14232            let result = runner.implement().await;
14233            assert_eq!(result.is_err(), block_rescue);
14234            assert_eq!(
14235                runner.state.resource_block.as_ref().unwrap().kind,
14236                "disk pressure"
14237            );
14238            assert!(
14239                runner
14240                    .state
14241                    .events
14242                    .iter()
14243                    .any(|e| e.message.contains("agent execution interrupted"))
14244            );
14245            assert!(
14246                runner
14247                    .state
14248                    .dir()
14249                    .join("artifacts/candidate-A-recovery/staged.patch")
14250                    .exists()
14251            );
14252            if block_rescue {
14253                let saved = RunState::load(&runner.state.id).unwrap();
14254                assert!(saved.candidate_preservation.contains_key(&'A'));
14255                assert!(!saved.candidates[0].empty);
14256                assert_eq!(
14257                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14258                    "retain implementation work\n"
14259                );
14260                std::fs::remove_file(lock).unwrap();
14261                runner.state = saved;
14262            } else {
14263                assert_eq!(runner.state.status, RunStatus::Blocked);
14264                assert_eq!(runner.state.candidates[0].commits, 1);
14265            }
14266            // Recovery must save the candidate before deciding whether another
14267            // implementation is needed. No implementer is available on retry.
14268            runner.roles.implementers.clear();
14269            runner.state.config.disk.min_free_bytes = 0;
14270            runner.state.resource_block = None;
14271            runner.implement().await.unwrap();
14272            assert!(runner.state.candidate_preservation.is_empty());
14273            assert_eq!(runner.state.candidates[0].commits, 1);
14274            assert!(!runner.state.candidates[0].empty);
14275            assert!(git::is_clean(&wt).await.unwrap());
14276            assert!(
14277                crate::run::read_artifact(&runner.state, "cand-A.patch")
14278                    .unwrap()
14279                    .contains("retain implementation work")
14280            );
14281        }
14282    }
14283
14284    #[tokio::test]
14285    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14286        let (_tmp, mut runner) = preservation_resume_fixture().await;
14287        // Schema 15 already confirmed the unfolded no-change result.
14288        runner.implement().await.unwrap();
14289        assert!(runner.state.candidate_preservation.is_empty());
14290        assert!(
14291            !runner
14292                .state
14293                .dir()
14294                .join("artifacts/candidate-A-recovery")
14295                .exists()
14296        );
14297        runner.fold_losers().await.unwrap();
14298        assert!(!runner.state.candidates[0].worktree.exists());
14299        for schema in [14, crate::run::SCHEMA] {
14300            runner.state.schema = schema;
14301            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14302            runner.state.save().unwrap();
14303            runner.state = RunState::load(&runner.state.id).unwrap();
14304            runner.implement().await.unwrap();
14305            assert!(runner.state.candidate_preservation.is_empty());
14306            assert!(runner.state.candidates[0].folded);
14307            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14308        }
14309    }
14310
14311    #[tokio::test]
14312    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14313        let (_tmp, mut runner) = preservation_resume_fixture().await;
14314        runner.state.candidates.retain(|c| c.label == 'A');
14315        runner.state.tally = None;
14316        runner.state.judge_skipped = false;
14317        let candidate = &mut runner.state.candidates[0];
14318        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14319        candidate.failed = Some("deployment validation unavailable".into());
14320        std::fs::write(
14321            candidate.worktree.join("repair"),
14322            "future launcher policy\n",
14323        )
14324        .unwrap();
14325        runner
14326            .state
14327            .candidate_preservation
14328            .insert('A', "awaiting persistence".into());
14329        runner.persist_candidates().await.unwrap();
14330        let candidate = &runner.state.candidates[0];
14331        assert!(!candidate.empty);
14332        assert_eq!(candidate.commits, 1);
14333        assert_eq!(candidate.files, 1);
14334        assert!(candidate.failed.is_none());
14335        assert!(candidate.summary.contains("EPERM"));
14336        assert!(
14337            crate::run::read_artifact(&runner.state, "cand-A.patch")
14338                .unwrap()
14339                .contains("future launcher policy")
14340        );
14341        assert!(runner.state.candidate_preservation.is_empty());
14342    }
14343
14344    #[tokio::test]
14345    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14346        let (_tmp, mut runner) = preservation_resume_fixture().await;
14347        let wt = runner.state.candidates[0].worktree.clone();
14348        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14349        runner.state.schema = 14;
14350        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14351        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14352        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14353        for _ in 0..2 {
14354            assert!(runner.implement().await.is_err());
14355            assert_eq!(runner.state.status, RunStatus::Blocked);
14356            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14357            assert!(
14358                runner.state.candidates[0].empty,
14359                "old verdict must remain unusable"
14360            );
14361            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14362            assert!(runner.state.judge_skipped);
14363            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14364            assert!(
14365                crate::run::read_artifact(&runner.state, "cand-A.patch")
14366                    .unwrap()
14367                    .contains("owned edits")
14368            );
14369            assert!(wt.exists());
14370            assert_eq!(
14371                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14372                    .await
14373                    .unwrap(),
14374                1
14375            );
14376            runner.state = RunState::load(&runner.state.id).unwrap();
14377        }
14378    }
14379
14380    #[tokio::test]
14381    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14382        let (_tmp, mut runner) = preservation_resume_fixture().await;
14383        let wt = runner.state.candidates[0].worktree.clone();
14384        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14385        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14386        let index_lock = PathBuf::from(
14387            git::git(
14388                &wt,
14389                &[
14390                    "rev-parse",
14391                    "--path-format=absolute",
14392                    "--git-path",
14393                    "index.lock",
14394                ],
14395            )
14396            .await
14397            .unwrap(),
14398        );
14399        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14400        runner.state.schema = 14;
14401        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14402        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14403        assert!(runner.implement().await.is_err());
14404        assert!(runner.after_implement().is_err());
14405        assert!(runner.fold_losers().await.is_err());
14406        assert_eq!(
14407            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14408            "failed seat edits"
14409        );
14410        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14411        runner.state = RunState::load(&runner.state.id).unwrap();
14412        std::fs::remove_file(index_lock).unwrap();
14413        // Recovery finds changes that the old uncontested result never judged.
14414        assert!(runner.implement().await.is_err());
14415        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14416        assert!(runner.state.candidates[0].empty);
14417        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14418        assert!(
14419            crate::run::read_artifact(&runner.state, "cand-A.patch")
14420                .unwrap()
14421                .contains("failed seat edits")
14422        );
14423        assert!(wt.exists());
14424    }
14425
14426    #[tokio::test]
14427    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14428        let (_tmp, mut runner) = preservation_resume_fixture().await;
14429        let repo = runner.state.repo.clone();
14430        let common = git::git(
14431            &repo,
14432            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14433        )
14434        .await
14435        .unwrap();
14436        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14437        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14438        assert!(runner.fold_losers().await.is_err());
14439        assert!(!runner.state.candidates[0].worktree.exists());
14440        assert!(!runner.state.candidates[0].folded);
14441        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14442        runner.state = RunState::load(&runner.state.id).unwrap();
14443        std::fs::remove_file(branch_lock).unwrap();
14444        let judge = runner.state.worktree_root().join("judge-1");
14445        git::worktree_add_detached(&repo, &judge, "main")
14446            .await
14447            .unwrap();
14448        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14449        runner.roles.judges = vec![spec("judge")];
14450        assert!(runner.fold_losers().await.is_err());
14451        runner.state = RunState::load(&runner.state.id).unwrap();
14452        assert!(
14453            runner.state.candidates[0].folded,
14454            "candidate progress saved before judge failure"
14455        );
14456        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14457        assert!(judge.join("uncommitted").exists());
14458        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14459        runner.fold_losers().await.unwrap();
14460        assert!(!judge.exists());
14461        runner.fold_losers().await.unwrap();
14462    }
14463
14464    #[tokio::test]
14465    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14466        let (_tmp, mut runner) = preservation_resume_fixture().await;
14467        runner.state.candidates.retain(|c| c.label == 'B');
14468        runner.state.tally = None;
14469        runner.state.judge_skipped = false;
14470        runner.state.prep_complete = Some(true);
14471        runner.state.config.graph.advise = false;
14472        runner.state.config.graph.selected_implementer = Some("alpha".into());
14473        let wt = runner.state.candidates[0].worktree.clone();
14474        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14475        runner
14476            .state
14477            .candidate_preservation
14478            .insert('B', "awaiting persistence".into());
14479        let index_lock = git::git(
14480            &wt,
14481            &[
14482                "rev-parse",
14483                "--path-format=absolute",
14484                "--git-path",
14485                "index.lock",
14486            ],
14487        )
14488        .await
14489        .unwrap();
14490        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14491        runner.state.save().unwrap();
14492        assert!(runner.execute_graph().await.is_err());
14493        assert_eq!(runner.state.status, RunStatus::Blocked);
14494        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14495        std::fs::remove_file(index_lock).unwrap();
14496        // Park at the next graph boundary: this tests the real selected-seat
14497        // hold guard without dispatching reviewers or publishing anything.
14498        for _ in 0..2 {
14499            runner.state = RunState::load(&runner.state.id).unwrap();
14500            runner.pause.park();
14501            runner.execute_graph().await.unwrap();
14502            assert_eq!(runner.state.status, RunStatus::Implementing);
14503            assert!(runner.state.candidate_preservation.is_empty());
14504            assert_eq!(runner.state.candidates[0].commits, 2);
14505            assert!(runner.state.tally.is_none());
14506        }
14507        runner.implement().await.unwrap();
14508        assert_eq!(runner.state.status, RunStatus::Judging);
14509        // A genuine selected-implementer hold must still require release.
14510        runner.state.status = RunStatus::Blocked;
14511        runner.state.candidates[0].failed = Some("implementation failed".into());
14512        runner.state.save().unwrap();
14513        let error = runner.execute_graph().await.unwrap_err();
14514        assert!(
14515            error
14516                .to_string()
14517                .contains("selected implementation is held")
14518        );
14519    }
14520
14521    #[tokio::test]
14522    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14523        let (_tmp, mut runner) = preservation_resume_fixture().await;
14524        let repo = runner.state.repo.clone();
14525        let candidate = runner.state.candidates[0].clone();
14526        // Remove only the disposable fixture directory, leaving Git's linked
14527        // worktree registration and branch intact, as in a vanished checkout.
14528        let registered = git::worktree_holding(&repo, &candidate.branch)
14529            .await
14530            .unwrap()
14531            .unwrap();
14532        assert_eq!(
14533            registered.canonicalize().unwrap(),
14534            candidate.worktree.canonicalize().unwrap()
14535        );
14536        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14537        assert_eq!(
14538            git::worktree_holding(&repo, &candidate.branch)
14539                .await
14540                .unwrap(),
14541            Some(registered)
14542        );
14543        assert!(
14544            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14545            "the stale registration initially keeps the branch checked out"
14546        );
14547        runner.fold_losers().await.unwrap();
14548        assert!(runner.state.candidates[0].folded);
14549        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14550        assert_eq!(
14551            git::worktree_holding(&repo, &candidate.branch)
14552                .await
14553                .unwrap(),
14554            None
14555        );
14556        runner.state = RunState::load(&runner.state.id).unwrap();
14557        runner.fold_losers().await.unwrap();
14558    }
14559
14560    #[tokio::test]
14561    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14562        let (_tmp, mut runner) = preservation_resume_fixture().await;
14563        let repo = runner.state.repo.clone();
14564        let candidate = runner.state.candidates[0].clone();
14565        let winner = runner.state.candidates[1].clone();
14566        let registered = git::worktree_holding(&repo, &candidate.branch)
14567            .await
14568            .unwrap()
14569            .unwrap();
14570        assert_eq!(
14571            registered.canonicalize().unwrap(),
14572            candidate.worktree.canonicalize().unwrap()
14573        );
14574        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14575        assert_eq!(
14576            git::worktree_holding(&repo, &candidate.branch)
14577                .await
14578                .unwrap(),
14579            Some(registered)
14580        );
14581        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14582
14583        for _ in 0..2 {
14584            fold_run(&mut runner.state, false, &crate::run::home())
14585                .await
14586                .unwrap();
14587            runner.state = RunState::load(&runner.state.id).unwrap();
14588            assert!(runner.state.candidates[0].folded);
14589            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14590            assert_eq!(
14591                git::worktree_holding(&repo, &candidate.branch)
14592                    .await
14593                    .unwrap(),
14594                None
14595            );
14596            assert!(!runner.state.candidates[1].folded);
14597            assert!(winner.worktree.exists());
14598            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14599        }
14600    }
14601
14602    #[tokio::test]
14603    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14604        ask_test_home();
14605        let tmp = tempfile::tempdir().unwrap();
14606        let repo = tmp.path().join("repo");
14607        std::fs::create_dir(&repo).unwrap();
14608        init_repo(&repo);
14609        let mut runner = runner_at(RunStatus::Implementing);
14610        runner.state.repo = repo.clone();
14611        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14612        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14613        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14614            let branch = format!("candidate-{label}");
14615            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14616            git::worktree_add_branch(&repo, &wt, &branch, "main")
14617                .await
14618                .unwrap();
14619            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14620            git::git(&wt, &["add", "README.md"]).await.unwrap();
14621            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14622            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14623            runner.state.candidates.push(Candidate {
14624                index,
14625                label,
14626                agent: "alpha".to_owned(),
14627                branch,
14628                worktree: wt,
14629                summary: "completed implementation".to_owned(),
14630                stat: String::new(),
14631                files: 0,
14632                commits: 0,
14633                empty: false,
14634                failed: None,
14635                verified_noop: None,
14636                duration_ms: 42,
14637                folded: false,
14638            });
14639            runner
14640                .state
14641                .candidate_preservation
14642                .insert(label, "awaiting persistence".to_owned());
14643        }
14644        let wt = runner.state.candidates[0].worktree.clone();
14645        let index_lock = git::git(
14646            &wt,
14647            &[
14648                "rev-parse",
14649                "--path-format=absolute",
14650                "--git-path",
14651                "index.lock",
14652            ],
14653        )
14654        .await
14655        .unwrap();
14656        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14657        runner.state.save().unwrap();
14658        assert!(runner.persist_candidates().await.is_err());
14659        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14660        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14661        assert_eq!(
14662            runner.state.candidates[1].commits, 1,
14663            "other results are still collected"
14664        );
14665        assert!(
14666            !runner.state.candidates[0].empty,
14667            "failed rescue is not no change"
14668        );
14669        assert_eq!(
14670            runner.state.candidates[0].summary,
14671            "completed implementation"
14672        );
14673        assert!(runner.after_implement().is_err());
14674        let home = tmp.path().join("fold-home");
14675        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14676        assert!(wt.join("new-file").exists());
14677        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14678        assert!(recovery.join("staged.patch").exists());
14679        assert!(recovery.join("unstaged.patch").exists());
14680        assert_eq!(
14681            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14682            "untracked edits\n"
14683        );
14684        runner.state.begin_pass();
14685        runner.state.end_pass_parked(ParkCause::Upgrade {
14686            from: "1.0".into(),
14687            to: Some("1.1".into()),
14688        });
14689        runner.state.save().unwrap();
14690        runner.state = RunState::load(&runner.state.id).unwrap();
14691        assert!(matches!(
14692            runner.state.passes.last().unwrap().end,
14693            Some(crate::run::PassEnd::Parked {
14694                cause: ParkCause::Upgrade { .. }
14695            })
14696        ));
14697        std::fs::remove_file(index_lock).unwrap();
14698        // No implementer is configured: a persistence retry cannot dispatch a seat.
14699        runner.implement().await.unwrap();
14700        assert!(runner.state.candidate_preservation.is_empty());
14701        assert!(
14702            runner
14703                .state
14704                .candidates
14705                .iter()
14706                .all(|c| c.commits == 1 && !c.empty)
14707        );
14708        assert!(git::is_clean(&wt).await.unwrap());
14709        assert!(
14710            crate::run::read_artifact(&runner.state, "cand-A.patch")
14711                .unwrap()
14712                .contains("later tracked edits")
14713        );
14714    }
14715
14716    #[tokio::test]
14717    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14718        ask_test_home();
14719        let tmp = tempfile::tempdir().unwrap();
14720        let repo = tmp.path().join("repo");
14721        std::fs::create_dir(&repo).unwrap();
14722        init_repo(&repo);
14723        let wt = tmp.path().join("candidate");
14724        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14725            .await
14726            .unwrap();
14727        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14728        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14729        assert!(wt.join("uncommitted").exists());
14730        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14731        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14732        let mut runner = runner_at(RunStatus::Implementing);
14733        runner.state.repo = repo.clone();
14734        runner.state.base_commit = "missing-base".to_owned();
14735        runner.state.candidates.push(Candidate {
14736            index: 0,
14737            label: 'A',
14738            agent: "alpha".into(),
14739            branch: "candidate".into(),
14740            worktree: wt.clone(),
14741            summary: "completed".into(),
14742            stat: String::new(),
14743            files: 0,
14744            commits: 0,
14745            empty: false,
14746            failed: None,
14747            verified_noop: None,
14748            duration_ms: 1,
14749            folded: false,
14750        });
14751        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14752        assert!(
14753            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14754                .await
14755                .is_err()
14756        );
14757        assert!(!runner.state.candidates[0].folded);
14758        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14759        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14760        runner
14761            .state
14762            .candidate_preservation
14763            .insert('A', "pending".into());
14764        assert!(runner.persist_candidates().await.is_err());
14765        assert!(!runner.state.candidates[0].empty);
14766        assert!(!runner.state.candidates[0].folded);
14767        assert!(wt.exists());
14768        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14769        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14770        std::fs::create_dir_all(&patch_path).unwrap();
14771        assert!(
14772            runner.persist_candidates().await.is_err(),
14773            "artifact failure stays pending"
14774        );
14775        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14776        assert!(!runner.state.candidates[0].empty);
14777        std::fs::remove_dir(patch_path).unwrap();
14778        runner.persist_candidates().await.unwrap();
14779        assert!(
14780            runner.state.candidates[0].empty,
14781            "only successful inspection confirms no change"
14782        );
14783    }
14784
14785    fn init_repo(dir: &Path) {
14786        let run = |args: &[&str]| {
14787            let out = std::process::Command::new("git")
14788                .args(args)
14789                .current_dir(dir)
14790                .quiet()
14791                .output()
14792                .expect("spawn git");
14793            assert!(
14794                out.status.success(),
14795                "git {args:?} failed: {}",
14796                String::from_utf8_lossy(&out.stderr)
14797            );
14798        };
14799        run(&["init", "-b", "main"]);
14800        run(&["config", "user.name", "magi test"]);
14801        run(&["config", "user.email", "magi@example.com"]);
14802        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14803        run(&["add", "-A"]);
14804        run(&["commit", "-m", "init"]);
14805    }
14806
14807    // `settle_questions` is what closes the ghost the phone showed: a run's
14808    // seat asked something, the run then ended, and nothing was left to
14809    // abandon the question it left `open`. `HOME` is a process-wide
14810    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14811    // first time it runs in the binary — every test below still reaches the
14812    // same directory whichever call won, and each gets its own run id from
14813    // `RunState::new`, so they never collide there.
14814    fn ask_test_home() {
14815        crate::run::pin_test_home();
14816    }
14817
14818    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14819    /// reads nothing else off it.
14820    fn runner_at(status: RunStatus) -> Runner {
14821        let mut state = RunState::new(
14822            PathBuf::from("/nonexistent/repo"),
14823            "main".to_owned(),
14824            "deadbeef".to_owned(),
14825            "task".to_owned(),
14826            Config::default(),
14827        );
14828        // These fixtures share one home and save concurrently. Short clock
14829        // IDs can collide, making an ownership test read another test's run.
14830        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14831        state.id = format!(
14832            "20000101-000000-{:04x}",
14833            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14834        );
14835        state.status = status;
14836        Runner {
14837            state,
14838            roles: ResolvedRoles {
14839                implementers: Vec::new(),
14840                judges: Vec::new(),
14841                reviewers: Vec::new(),
14842                fixer: None,
14843                conductor: conductor(),
14844                implementer_roster: Vec::new(),
14845                judge_roster: Vec::new(),
14846                reviewer_roster: Vec::new(),
14847            },
14848            sem: Arc::new(Semaphore::new(1)),
14849            pause: Pause::new(),
14850            interrupt: Pause::new(),
14851        }
14852    }
14853
14854    #[tokio::test]
14855    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14856        crate::run::pin_test_home();
14857        let mut runner = runner_at(RunStatus::Gating);
14858        runner.state.save().unwrap();
14859        let id = runner.state.id.clone();
14860        let passes = runner.state.passes.len();
14861        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14862        let mut claiming = Box::pin(runner.claim_driver());
14863        std::future::poll_fn(|cx| {
14864            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14865            std::task::Poll::Ready(())
14866        })
14867        .await;
14868        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14869        drop(owner);
14870        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14871            .await
14872            .unwrap()
14873            .unwrap();
14874        assert_eq!(execution.runner.state.passes.len(), passes + 1);
14875        assert!(execution.runner.state.resource_block.is_none());
14876        assert_eq!(
14877            execution.runner.state.liveness(false),
14878            crate::run::Liveness::Live
14879        );
14880        drop(execution);
14881        assert!(RunState::load(&id).unwrap().driver_exited);
14882    }
14883
14884    #[tokio::test]
14885    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14886        crate::run::pin_test_home();
14887        let mut runner = runner_at(RunStatus::Gating);
14888        runner.state.save().unwrap();
14889        // run.json can commit, but the subsequent lease metadata write fails.
14890        // Cleanup must run after releasing the short ownership lock.
14891        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14892        assert!(runner.claim_driver().await.is_err());
14893        assert!(runner.state.driver_exited);
14894        assert_eq!(runner.state.driver_pid, None);
14895        let disk = RunState::load(&runner.state.id).unwrap();
14896        assert!(disk.driver_exited);
14897        assert_eq!(disk.driver_pid, None);
14898        assert_eq!(disk.status, RunStatus::Gating);
14899    }
14900
14901    #[tokio::test]
14902    async fn an_early_graph_error_releases_its_driver() {
14903        crate::run::pin_test_home();
14904        let mut runner = runner_at(RunStatus::Blocked);
14905        runner.state.config.graph.selected_implementer = Some("selected".into());
14906        runner.state.save().unwrap();
14907        assert!(runner.execute().await.is_err());
14908        let disk = RunState::load(&runner.state.id).unwrap();
14909        assert!(disk.driver_exited);
14910        assert_eq!(disk.driver_pid, None);
14911        assert_eq!(disk.status, RunStatus::Blocked);
14912    }
14913
14914    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14915        crate::run::MergeRepair {
14916            question: "repair-question".into(),
14917            head: "old-head".into(),
14918            pr: "https://github.com/o/r/pull/42".into(),
14919            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14920            gate: vec![outcome(Some(0), false)],
14921            gate_head: Some("old-head".into()),
14922            gate_fixes: Vec::new(),
14923            pre_gate: Vec::new(),
14924            phase,
14925            failure: None,
14926        }
14927    }
14928
14929    #[tokio::test]
14930    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14931        crate::run::pin_test_home();
14932        let mut runner = runner_at(RunStatus::Landing);
14933        runner.state.merge_repairs.push(merge_repair_request(
14934            crate::run::MergeRepairPhase::Requested,
14935        ));
14936        runner.state.save().unwrap();
14937        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14938        assert!(
14939            runner
14940                .execute()
14941                .await
14942                .unwrap_err()
14943                .to_string()
14944                .contains("already running")
14945        );
14946        assert_eq!(
14947            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14948            crate::run::MergeRepairPhase::Requested
14949        );
14950        drop(claim);
14951        let mut disk = runner.state.clone();
14952        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14953        disk.save().unwrap();
14954        assert!(
14955            runner
14956                .execute()
14957                .await
14958                .unwrap_err()
14959                .to_string()
14960                .contains("decision changed")
14961        );
14962        assert_eq!(
14963            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14964            crate::run::MergeRepairPhase::Complete
14965        );
14966    }
14967
14968    #[test]
14969    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14970        crate::run::pin_test_home();
14971        let mut runner = runner_at(RunStatus::Landing);
14972        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14973        runner.state.reviews = request.reviews.clone();
14974        runner.state.gate = request.gate.clone();
14975        runner.state.gate_ran = true;
14976        runner.state.land_approval = Some(crate::run::LandApproval {
14977            question: "old".into(),
14978            head: "old-head".into(),
14979        });
14980        runner.state.merge_repairs.push(request);
14981        runner.begin_merge_repair_review(0).unwrap();
14982        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14983        assert!(!runner.state.gate_status().ok());
14984        assert!(runner.state.land_approval.is_none());
14985        let disk = RunState::load(&runner.state.id).unwrap();
14986        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14987        assert!(disk.merge_repairs[0].gate[0].ok());
14988        assert_eq!(
14989            disk.merge_repairs[0].phase,
14990            crate::run::MergeRepairPhase::Reviewing
14991        );
14992        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14993        assert!(disk.reviews.is_empty());
14994    }
14995
14996    #[tokio::test]
14997    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14998        crate::run::pin_test_home();
14999        for phase in [
15000            crate::run::MergeRepairPhase::Fixing,
15001            crate::run::MergeRepairPhase::Requested,
15002        ] {
15003            let mut runner = runner_at(RunStatus::Landing);
15004            runner.state.config.graph.land_rounds = 0;
15005            runner.state.merge_repairs.push(merge_repair_request(phase));
15006            runner.state.save().unwrap();
15007            assert!(!runner.repair_merge_review().await.unwrap());
15008            assert_eq!(runner.state.status, RunStatus::Blocked);
15009            assert_eq!(
15010                runner.state.merge_repairs[0].phase,
15011                crate::run::MergeRepairPhase::Failed
15012            );
15013            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
15014            assert!(
15015                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
15016                    "interrupted"
15017                } else {
15018                    "budget exhausted"
15019                })
15020            );
15021            assert!(runner.state.merge.is_none());
15022        }
15023    }
15024
15025    /// `park_here` folding in the reason `Pause::park_because` recorded -
15026    /// this is what lets an operator reading a run's events tell an
15027    /// interrupt-driven park from an ordinary shutdown park.
15028    #[test]
15029    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15030        crate::run::pin_test_home();
15031        let mut runner = runner_at(RunStatus::Implementing);
15032        let interrupt = Pause::new();
15033        runner.watch_interrupt(interrupt.clone());
15034
15035        interrupt.park_because("task a1b2 asked to run first");
15036
15037        assert!(runner.park_here().expect("park_here"));
15038        assert!(runner.state.parked);
15039        let last = runner.state.events.last().expect("a park event");
15040        assert_eq!(last.node, "park");
15041        assert!(
15042            last.message.contains("task a1b2 asked to run first"),
15043            "expected the interrupt reason in {:?}",
15044            last.message
15045        );
15046    }
15047
15048    /// The cause is stored on the pass when it parks, and the first cause a
15049    /// shared pause was given is the one that stays.
15050    #[test]
15051    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15052        crate::run::pin_test_home();
15053        let mut runner = runner_at(RunStatus::Implementing);
15054        runner.state.begin_pass();
15055        let shutdown = Pause::new();
15056        runner.on_pause(shutdown.clone());
15057        let upgrade = ParkCause::Upgrade {
15058            from: "0.121.0".to_owned(),
15059            to: Some("0.121.1".to_owned()),
15060        };
15061        shutdown.park_for(upgrade.clone());
15062        shutdown.park_for(ParkCause::Operator);
15063
15064        assert!(runner.park_here().expect("park_here"));
15065        assert_eq!(
15066            runner.state.passes[0].end,
15067            Some(crate::run::PassEnd::Parked { cause: upgrade })
15068        );
15069        let last = runner.state.events.last().expect("a park event");
15070        assert!(last.message.contains("0.121.0"), "{}", last.message);
15071    }
15072
15073    #[tokio::test]
15074    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15075        crate::run::pin_test_home();
15076        let mut runner = runner_at(RunStatus::Reviewing);
15077        runner.state.save().unwrap();
15078        let mut execution = runner.claim_driver().await.expect("first execution");
15079        assert_eq!(execution.runner.state.passes.len(), 1);
15080        execution
15081            .driver
15082            .finish(&mut execution.runner.state)
15083            .await
15084            .unwrap();
15085    }
15086
15087    #[tokio::test]
15088    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15089        crate::run::pin_test_home();
15090        let mut runner = runner_at(RunStatus::Reviewing);
15091        runner.state.config.agents = vec![conductor()];
15092        runner.state.config.graph.timeout_resource_wait = 0;
15093        runner.state.save().unwrap();
15094        let id = runner.state.id.clone();
15095        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15096        assert!(runner.claim_driver().await.is_err());
15097        drop(owner);
15098        drop(runner);
15099        let mut resumed = Runner::resume(&id).unwrap();
15100        assert_eq!(resumed.state.driver_claimed, Some(false));
15101        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15102        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15103        let mut execution = resumed
15104            .claim_driver()
15105            .await
15106            .expect("retry initial claim after reload");
15107        assert_eq!(execution.runner.state.driver_claimed, Some(true));
15108        assert_eq!(execution.runner.state.passes.len(), 1);
15109        execution
15110            .driver
15111            .finish(&mut execution.runner.state)
15112            .await
15113            .unwrap();
15114    }
15115
15116    #[tokio::test]
15117    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15118        crate::run::pin_test_home();
15119        let mut runner = runner_at(RunStatus::Reviewing);
15120        runner.state.config.agents = vec![conductor()];
15121        runner.state.save().unwrap();
15122        let id = runner.state.id.clone();
15123        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15124        let mut resumed = Runner::resume(&id).unwrap();
15125        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15126        assert!(resumed.claim_driver().await.is_err());
15127        assert!(RunState::load(&id).unwrap().passes.is_empty());
15128        drop(lease);
15129        let mut execution = resumed.claim_driver().await.unwrap();
15130        execution
15131            .driver
15132            .finish(&mut execution.runner.state)
15133            .await
15134            .unwrap();
15135    }
15136
15137    #[tokio::test]
15138    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15139        crate::run::pin_test_home();
15140        let mut runner = runner_at(RunStatus::Reviewing);
15141        let state = &mut runner.state;
15142        state.config.agents = vec![conductor()];
15143        state.driver_claimed = None;
15144        state.driver_pid = Some(std::process::id());
15145        state.driver_started_at = None;
15146        state.begin_pass();
15147        state.save().unwrap();
15148        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15149        std::fs::write(
15150            state.dir().join("driver.json"),
15151            serde_json::to_vec(&serde_json::json!({
15152                "pid": state.driver_pid, "started_at": null,
15153                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15154            }))
15155            .unwrap(),
15156        )
15157        .unwrap();
15158        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15159        drop(lease);
15160        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15161        let mut resumed = Runner::resume(&state.id).unwrap();
15162        assert!(!resumed.state.driver_exited);
15163        let mut execution = resumed
15164            .claim_driver()
15165            .await
15166            .expect("lease proves previous execution ended");
15167        assert_eq!(execution.runner.state.passes.len(), 2);
15168        execution
15169            .driver
15170            .finish(&mut execution.runner.state)
15171            .await
15172            .unwrap();
15173    }
15174
15175    #[tokio::test]
15176    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15177        crate::run::pin_test_home();
15178        let mut runner = runner_at(RunStatus::Reviewing);
15179        let mut json = serde_json::to_value(&runner.state).unwrap();
15180        json.as_object_mut().unwrap().remove("driver_claimed");
15181        runner.state = serde_json::from_value(json).unwrap();
15182        assert_eq!(runner.state.driver_claimed, None);
15183        runner.state.save().unwrap();
15184        let mut execution = runner
15185            .claim_driver()
15186            .await
15187            .expect("the execution lease, not a missing marker, excludes a second driver");
15188        assert_eq!(execution.runner.state.passes.len(), 1);
15189        execution
15190            .driver
15191            .finish(&mut execution.runner.state)
15192            .await
15193            .unwrap();
15194    }
15195
15196    #[tokio::test]
15197    async fn driver_admission_refuses_a_proven_live_driver() {
15198        crate::run::pin_test_home();
15199        let mut runner = runner_at(RunStatus::Reviewing);
15200        let pid = std::process::id();
15201        runner.state.driver_pid = Some(pid);
15202        runner.state.driver_started_at = crate::proc::process_started_at(pid);
15203        assert!(runner.state.driver_started_at.is_some());
15204        runner.state.save().unwrap();
15205        let error = runner
15206            .claim_driver()
15207            .await
15208            .err()
15209            .expect("a live driver must refuse");
15210        assert!(error.to_string().contains("live driver"), "{error:#}");
15211        let saved = RunState::load(&runner.state.id).unwrap();
15212        assert_eq!(saved.driver_pid, Some(pid));
15213        assert_eq!(saved.status, RunStatus::Reviewing);
15214        assert!(saved.passes.is_empty());
15215    }
15216
15217    #[test]
15218    fn parking_resource_halt_persists_the_pass_once() {
15219        crate::run::pin_test_home();
15220        let mut runner = runner_at(RunStatus::Reviewing);
15221        runner.state.begin_pass();
15222        runner.state.resource_block = Some(crate::run::ResourceBlock {
15223            node: "fix".into(),
15224            kind: "shared build cache".into(),
15225            detail: "cancelled".into(),
15226            at: Timestamp::now(),
15227            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15228        });
15229        let cause = ParkCause::Upgrade {
15230            from: "old".into(),
15231            to: Some("new".into()),
15232        };
15233        runner.pause.park_for(cause.clone());
15234        // Both direct fixer exits and graph resource boundaries must complete
15235        // parking, rather than returning a stale Reviewing record.
15236        runner.stop_for_resource().unwrap();
15237        assert!(runner.resource_halt().unwrap());
15238        assert!(runner.park_here().unwrap());
15239        let saved = RunState::load(&runner.state.id).unwrap();
15240        assert!(saved.parked);
15241        assert_eq!(saved.status, RunStatus::Reviewing);
15242        assert!(saved.passes[0].ended_at.is_some());
15243        assert_eq!(
15244            saved.passes[0].end,
15245            Some(crate::run::PassEnd::Parked {
15246                cause: cause.clone()
15247            })
15248        );
15249        assert_eq!(
15250            saved.resource_block.unwrap().cause,
15251            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15252        );
15253        assert_eq!(
15254            saved
15255                .events
15256                .iter()
15257                .filter(|event| event.node == "park")
15258                .count(),
15259            1
15260        );
15261    }
15262
15263    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
15264    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
15265    /// cleared) must not make a *different* run - one only watching its own,
15266    /// unshared interrupt `Pause` - see itself as parked. If a future change
15267    /// ever collapsed these back into one handle, the interrupt scheduler
15268    /// would park every run for the rest of the daemon's life, not just the
15269    /// one it meant to interrupt.
15270    #[test]
15271    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15272        crate::run::pin_test_home();
15273        let mut runner = runner_at(RunStatus::Implementing);
15274        let shutdown = Pause::new();
15275        runner.on_pause(shutdown.clone());
15276        let interrupt = Pause::new();
15277        runner.watch_interrupt(interrupt.clone());
15278
15279        // Nobody has asked for anything yet.
15280        assert!(!runner.park_here().expect("park_here"));
15281        assert!(!runner.state.parked);
15282
15283        // Only the interrupt handle fires; the shutdown handle stays clear.
15284        interrupt.park_because("test");
15285        assert!(!shutdown.parked());
15286        assert!(runner.park_here().expect("park_here"));
15287    }
15288
15289    /// The property every prior attempt at this feature failed to pin down:
15290    /// asking a run to park while one of its nodes has a real, in-flight
15291    /// async operation running (an agent call, in production) must not cut
15292    /// that operation short. `park_here` is only ever consulted *between*
15293    /// `execute`'s node calls - see its own doc - so nothing inside a node
15294    /// can observe a park request until the node itself returns. This proves
15295    /// that structurally, with real `tokio` concurrency and a channel
15296    /// handshake (never a sleep, which would only prove "usually", not
15297    /// "cannot"): the "node" below reports that it has genuinely started,
15298    /// and only then is the park requested; the node still has to be told to
15299    /// finish before `park_here` is ever called, exactly mirroring every
15300    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
15301    /// in `execute`.
15302    #[tokio::test]
15303    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15304        crate::run::pin_test_home();
15305        let mut runner = runner_at(RunStatus::Implementing);
15306        let interrupt = Pause::new();
15307        runner.watch_interrupt(interrupt.clone());
15308
15309        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15310        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15311
15312        // Stands in for one node's in-flight agent call: it proves it has
15313        // genuinely started, then blocks - exactly as a spawned CLI process
15314        // does - until told to finish.
15315        let node = async move {
15316            started_tx.send(()).expect("send started");
15317            finish_rx.await.expect("recv finish");
15318            "node finished"
15319        };
15320
15321        let interrupter = async move {
15322            started_rx.await.expect("recv started");
15323            // The call is now genuinely in flight. Ask it to park.
15324            interrupt.park_because("higher-priority task waiting");
15325            // Nothing the node does can observe this yet - there is no
15326            // check inside it, by construction - so let the executor run
15327            // anything pending and then let the node finish on its own.
15328            tokio::task::yield_now().await;
15329            finish_tx.send(()).expect("send finish");
15330        };
15331
15332        let (node_result, ()) = tokio::join!(node, interrupter);
15333        assert_eq!(
15334            node_result, "node finished",
15335            "the in-flight call ran to completion"
15336        );
15337
15338        // Only now, at the boundary the real `execute` would check right
15339        // after this node, does the park take effect.
15340        assert!(runner.park_here().expect("park_here"));
15341        assert!(runner.state.parked);
15342    }
15343
15344    /// A run parked mid-competition carries every field it had accumulated
15345    /// through the exact same disk round-trip an ordinary resume uses -
15346    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
15347    /// Nothing about parking for an interrupt is a special case of that path;
15348    /// this is what proves it rather than assuming it.
15349    #[test]
15350    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15351        crate::run::pin_test_home();
15352        let mut runner = runner_at(RunStatus::Judging);
15353        // `Runner::resume` re-resolves roles from the saved config, which
15354        // refuses an empty roster - give it the same minimal one `conductor`
15355        // itself uses.
15356        runner.state.config.agents = vec![conductor()];
15357        runner.state.candidates = vec![Candidate {
15358            index: 0,
15359            label: 'A',
15360            agent: "alpha".to_owned(),
15361            branch: "magi/x/A".to_owned(),
15362            worktree: PathBuf::from("/nonexistent/worktree"),
15363            summary: "did the thing".to_owned(),
15364            stat: "1 file changed".to_owned(),
15365            files: 1,
15366            commits: 1,
15367            empty: false,
15368            failed: None,
15369            verified_noop: None,
15370            duration_ms: 1234,
15371            folded: false,
15372        }];
15373        let run_id = runner.state.id.clone();
15374
15375        let interrupt = Pause::new();
15376        runner.watch_interrupt(interrupt.clone());
15377        interrupt.park_because("task c3d4 asked to run first");
15378        assert!(runner.park_here().expect("park_here"));
15379
15380        let resumed = Runner::resume(&run_id).expect("resume");
15381        assert_eq!(resumed.state.candidates.len(), 1);
15382        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15383        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15384        assert_eq!(resumed.state.status, runner.state.status);
15385        assert!(
15386            resumed.state.parked,
15387            "still parked until `execute` actually walks the graph again"
15388        );
15389        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15390    }
15391
15392    /// A fresh open question on `run`, stored and handed back for assertions.
15393    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15394        let mut q = ask::Question::new(
15395            run.to_owned(),
15396            "implement".to_owned(),
15397            "impl-A".to_owned(),
15398            "Which storage backend should the cache use?".to_owned(),
15399            String::new(),
15400            vec!["SQLite".to_owned(), "Redis".to_owned()],
15401        );
15402        store.put(&mut q).unwrap();
15403        q
15404    }
15405
15406    #[test]
15407    fn a_failed_runs_open_question_is_abandoned() {
15408        ask_test_home();
15409        let store = ask::Questions::open();
15410        let mut runner = runner_at(RunStatus::Failed);
15411        let run = runner.state.id.clone();
15412        let q = ask_open_question(&store, &run);
15413
15414        runner.settle_questions();
15415
15416        let back = store.get(&q.id).unwrap();
15417        assert!(
15418            !back.status.open(),
15419            "the seat that asked died with the run; nobody is left to read an answer"
15420        );
15421        assert!(
15422            back.detail.contains(&run) && back.detail.contains("failed"),
15423            "the reason names what the run became, not just that it is gone: {}",
15424            back.detail
15425        );
15426    }
15427
15428    #[test]
15429    fn a_merged_runs_open_question_is_abandoned_too() {
15430        ask_test_home();
15431        let store = ask::Questions::open();
15432        // A run that finishes cleanly still leaves nobody to read an answer -
15433        // this is not only a failure-path cleanup.
15434        for status in [RunStatus::Merged, RunStatus::Ready] {
15435            let mut runner = runner_at(status);
15436            let run = runner.state.id.clone();
15437            let q = ask_open_question(&store, &run);
15438
15439            runner.settle_questions();
15440
15441            let back = store.get(&q.id).unwrap();
15442            assert!(
15443                !back.status.open(),
15444                "{status:?} run's question must not outlive the run"
15445            );
15446        }
15447    }
15448
15449    #[test]
15450    fn a_still_resumable_runs_open_question_is_left_alone() {
15451        ask_test_home();
15452        let store = ask::Questions::open();
15453        // `Blocked` and `Stalled` can still be resumed — the candidates, the
15454        // review round and the seat sessions are all still on disk — so a
15455        // question asked mid-round may yet get a real answer from a real
15456        // resume. Sweeping it here would be exactly the failure mode this
15457        // whole feature exists to avoid on the other side.
15458        for status in [RunStatus::Blocked, RunStatus::Stalled] {
15459            let mut runner = runner_at(status);
15460            let run = runner.state.id.clone();
15461            let q = ask_open_question(&store, &run);
15462
15463            runner.settle_questions();
15464
15465            let back = store.get(&q.id).unwrap();
15466            assert!(
15467                back.status.open(),
15468                "{status:?} is still alive; the question must still be waiting"
15469            );
15470        }
15471    }
15472
15473    #[test]
15474    fn settle_questions_never_touches_an_already_answered_question() {
15475        ask_test_home();
15476        let store = ask::Questions::open();
15477        let mut runner = runner_at(RunStatus::Failed);
15478        let run = runner.state.id.clone();
15479        let mut q = ask_open_question(&store, &run);
15480        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15481            .unwrap();
15482        store.put(&mut q).unwrap();
15483
15484        // Called twice, the way a crash-recovered daemon reclaim and the
15485        // graph's own cleanup both can for the same run — `abandon_for_run`
15486        // only ever touches what is still open, so this must be inert both
15487        // times, not merely the second.
15488        runner.settle_questions();
15489        runner.settle_questions();
15490
15491        let back = store.get(&q.id).unwrap();
15492        assert_eq!(
15493            back.status,
15494            ask::QuestionStatus::Answered,
15495            "a real answer is a decision on record, never overwritten by a sweep"
15496        );
15497    }
15498
15499    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15500    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15501    /// without merging, whose winner is still the operator's answer to read.
15502    /// Nothing previously called `fold_run` itself with a real `tally`, so
15503    /// this is the first test to pin down the one distinction the whole
15504    /// automatic-fold feature depends on: the winner's worktree and branch
15505    /// must survive, everything else sharing the run's worktree bay - a
15506    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15507    /// second sweep treats every non-winner directory under the bay alike -
15508    /// must not.
15509    #[tokio::test]
15510    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15511        crate::run::pin_test_home();
15512        let tmp = tempfile::tempdir().expect("tempdir");
15513        let repo = tmp.path().join("repo");
15514        std::fs::create_dir_all(&repo).unwrap();
15515        init_repo(&repo);
15516
15517        let mut config = Config::default();
15518        config.graph.worktree_root = Some(tmp.path().join("wt"));
15519
15520        let mut state = RunState::new(
15521            repo.clone(),
15522            "main".to_owned(),
15523            "deadbeef".to_owned(),
15524            "task".to_owned(),
15525            config,
15526        );
15527        let root = state.worktree_root();
15528        let wt_a = root.join("cand-A");
15529        let wt_b = root.join("cand-B");
15530        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15531            .await
15532            .expect("worktree A");
15533        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15534            .await
15535            .expect("worktree B");
15536
15537        state.candidates = vec![
15538            Candidate {
15539                index: 0,
15540                label: 'A',
15541                agent: "alpha".to_owned(),
15542                branch: "magi/x/A".to_owned(),
15543                worktree: wt_a.clone(),
15544                summary: String::new(),
15545                stat: String::new(),
15546                files: 0,
15547                commits: 0,
15548                empty: false,
15549                failed: None,
15550                verified_noop: None,
15551                duration_ms: 0,
15552                folded: false,
15553            },
15554            Candidate {
15555                index: 1,
15556                label: 'B',
15557                agent: "beta".to_owned(),
15558                branch: "magi/x/B".to_owned(),
15559                worktree: wt_b.clone(),
15560                summary: String::new(),
15561                stat: String::new(),
15562                files: 0,
15563                commits: 0,
15564                empty: false,
15565                failed: None,
15566                verified_noop: None,
15567                duration_ms: 0,
15568                folded: false,
15569            },
15570        ];
15571        state.tally = Some(Tally {
15572            first_choice: BTreeMap::from([('A', 1)]),
15573            borda: BTreeMap::new(),
15574            winner: 'A',
15575            rankings: 1,
15576            unanimous_initial: true,
15577            deliberated: false,
15578            changed_votes: 0,
15579            unanimous_final: true,
15580            tie_break: None,
15581            judges: 1,
15582            present: 1,
15583            quorum: 1,
15584            met_quorum: true,
15585            uncontested: None,
15586        });
15587        state.status = RunStatus::Ready;
15588
15589        fold_run(&mut state, false, &crate::run::home())
15590            .await
15591            .expect("fold_run");
15592
15593        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15594        assert!(
15595            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15596            "the unmerged winner's branch survives"
15597        );
15598        assert!(
15599            !state.candidates[0].folded,
15600            "the winner is not marked folded"
15601        );
15602
15603        assert!(!wt_b.exists(), "the loser's worktree is removed");
15604        assert!(
15605            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15606            "the loser's branch is removed"
15607        );
15608        assert!(state.candidates[1].folded, "the loser is marked folded");
15609    }
15610
15611    #[tokio::test]
15612    #[cfg(unix)]
15613    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15614        use std::os::fd::AsRawFd as _;
15615        use std::time::Duration;
15616
15617        let tmp = tempfile::tempdir().unwrap();
15618        let repo = tmp.path().join("repo");
15619        std::fs::create_dir_all(&repo).unwrap();
15620        init_repo(&repo);
15621        let common = repo.join(".git");
15622        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15623        let home = tmp.path().join("home");
15624        let mut config = Config::default();
15625        config.coordination = crate::config::test_coordination();
15626        config.graph.worktree_root = Some(tmp.path().join("wt"));
15627        config.graph.timeout_resource_wait = 1;
15628        let mut state = RunState::new(
15629            repo.clone(),
15630            "main".into(),
15631            "deadbeef".into(),
15632            "task".into(),
15633            config,
15634        );
15635        let root = state.worktree_root();
15636        let candidate = root.join("cand-A");
15637        let reviewer = root.join("review-1");
15638        let branch = state.branch_for('A');
15639        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15640            .await
15641            .unwrap();
15642        git::worktree_add_detached(&repo, &reviewer, "main")
15643            .await
15644            .unwrap();
15645        git::acquire_worktree_config(&repo).await.unwrap();
15646        state.enabled_worktree_config = true;
15647        state.status = RunStatus::Ready;
15648        state.candidates.push(Candidate {
15649            index: 0,
15650            label: 'A',
15651            agent: "implement".into(),
15652            branch: branch.clone(),
15653            worktree: candidate.clone(),
15654            summary: String::new(),
15655            stat: String::new(),
15656            files: 0,
15657            commits: 0,
15658            empty: false,
15659            failed: None,
15660            verified_noop: None,
15661            duration_ms: 0,
15662            folded: false,
15663        });
15664        state.save_under(&home).unwrap();
15665        let config_before = std::fs::read(common.join("config")).unwrap();
15666        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15667            .await
15668            .unwrap();
15669        // A separate OS handle bypasses the launcher's local gate, exactly as
15670        // an external updater does, and must still prevent all folding work.
15671        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15672        assert_eq!(
15673            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15674            0
15675        );
15676        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15677        assert!(
15678            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15679                .await
15680                .is_err()
15681        );
15682        assert!(candidate.exists());
15683        assert!(reviewer.exists());
15684        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15685        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15686        assert_eq!(
15687            git::git(&repo, &["worktree", "list", "--porcelain"])
15688                .await
15689                .unwrap(),
15690            worktrees_before
15691        );
15692        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15693            .await
15694            .unwrap()
15695            .unwrap_err();
15696        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15697        drop(fold);
15698        assert!(candidate.exists() && reviewer.exists());
15699        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15700        assert_eq!(
15701            state.resource_block.as_ref().unwrap().kind,
15702            "repository coordination"
15703        );
15704        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15705        drop(external);
15706        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15707        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15708            .await
15709            .unwrap()
15710            .unwrap();
15711        drop(fold);
15712        assert!(removed.contains(&branch));
15713        assert!(!root.exists());
15714        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15715        assert!(state.candidates[0].folded);
15716        assert!(!state.enabled_worktree_config);
15717        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15718        let enabled = git::git_raw(
15719            &repo,
15720            &["config", "--local", "--get", "extensions.worktreeConfig"],
15721        )
15722        .await
15723        .unwrap();
15724        assert!(!enabled.ok(), "the last config reference was released");
15725    }
15726
15727    /// A branch handed to a later run is that run's (and its pull request's):
15728    /// folding the run that released it must not delete it.
15729    #[tokio::test]
15730    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15731        let tmp = tempfile::tempdir().expect("tempdir");
15732        let repo = tmp.path().join("repo");
15733        std::fs::create_dir_all(&repo).unwrap();
15734        init_repo(&repo);
15735        let home = tmp.path().join("home");
15736
15737        let mut config = Config::default();
15738        config.graph.worktree_root = Some(tmp.path().join("wt"));
15739        let mut state = RunState::new(
15740            repo.clone(),
15741            "main".to_owned(),
15742            "deadbeef".to_owned(),
15743            "task".to_owned(),
15744            config,
15745        );
15746        // The worktree is already gone (released); the branch survives.
15747        git::git(&repo, &["branch", "magi/x/A", "main"])
15748            .await
15749            .expect("branch");
15750        state.candidates = vec![Candidate {
15751            index: 0,
15752            label: 'A',
15753            agent: "alpha".to_owned(),
15754            branch: "magi/x/A".to_owned(),
15755            worktree: state.worktree_root().join("cand-A"),
15756            summary: String::new(),
15757            stat: String::new(),
15758            files: 0,
15759            commits: 0,
15760            empty: false,
15761            failed: None,
15762            verified_noop: None,
15763            duration_ms: 0,
15764            folded: true,
15765        }];
15766        state.released_to = Some("20260901-000000-new1".to_owned());
15767        state.released_branches = vec!["magi/x/A".to_owned()];
15768
15769        fold_run(&mut state, true, &home).await.expect("fold_run");
15770        state.status = RunStatus::Merged;
15771        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15772        state.merge = Some(crate::run::MergeOutcome {
15773            mode: MergeMode::Pr,
15774            ok: true,
15775            detail: "confirmed merged".into(),
15776            empty: false,
15777        });
15778        fold_run(&mut state, true, &home)
15779            .await
15780            .expect("merged fold");
15781
15782        assert!(
15783            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15784            "the handed-over branch survives a fold"
15785        );
15786    }
15787
15788    /// A winner with nothing ahead of the base is caught before `gh` is ever
15789    /// asked for a pull request, and the message carries what the task's
15790    /// references resolved to.
15791    #[tokio::test]
15792    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15793        let tmp = tempfile::tempdir().expect("tempdir");
15794        let repo = tmp.path().join("repo");
15795        std::fs::create_dir_all(&repo).unwrap();
15796        init_repo(&repo);
15797        let run = |args: &[&str]| {
15798            let out = std::process::Command::new("git")
15799                .quiet()
15800                .args(args)
15801                .current_dir(&repo)
15802                .output()
15803                .expect("spawn git");
15804            assert!(out.status.success(), "git {args:?}");
15805        };
15806        run(&["branch", "magi/x/A"]);
15807        run(&["checkout", "-q", "-b", "magi/x/B"]);
15808        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15809        run(&["add", "-A"]);
15810        run(&["commit", "-q", "-m", "work"]);
15811        run(&["checkout", "-q", "main"]);
15812
15813        // A pull request is compared against the remote's base, so the
15814        // fixture needs one. Before it exists nothing can be read, and the
15815        // answer must be "not empty".
15816        let probe = RunState::new(
15817            repo.clone(),
15818            "main".to_owned(),
15819            "deadbeef".to_owned(),
15820            "task".to_owned(),
15821            Config::default(),
15822        );
15823        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15824        let bare = tmp.path().join("origin.git");
15825        let out = std::process::Command::new("git")
15826            .quiet()
15827            .args(["init", "-q", "--bare"])
15828            .arg(&bare)
15829            .output()
15830            .expect("spawn git");
15831        assert!(out.status.success(), "git init --bare");
15832        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15833        run(&["push", "-q", "origin", "main"]);
15834
15835        let mut state = RunState::new(
15836            repo.clone(),
15837            "main".to_owned(),
15838            "deadbeef".to_owned(),
15839            "task".to_owned(),
15840            Config::default(),
15841        );
15842        state.seeds = vec![refs::Seed {
15843            token: "magi/27b2/A".to_owned(),
15844            kind: refs::SeedKind::Unresolved,
15845            sha: String::new(),
15846            branch: true,
15847            detail: "no branch or commit named magi/27b2/A".to_owned(),
15848        }];
15849
15850        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15851        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15852        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15853        let detail = empty_candidate_detail(&state, "main");
15854        assert!(detail.starts_with("empty candidate"), "{detail}");
15855        assert!(!detail.contains("magi/27b2/A"), "{detail}");
15856        assert!(detail.contains("--start-from"), "{detail}");
15857    }
15858
15859    /// `status == Ready` used to be read as "this is the harmless
15860    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15861    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15862    /// run's PR was closed without merging — and reentering `merge` with
15863    /// `mode` still `Pr` does not know the difference, so it pushed and
15864    /// opened a second pull request. `mode == Local` reproduces the same
15865    /// blind spot without a network call: reentry must not attempt another
15866    /// git merge once this node has already recorded an outcome.
15867    #[tokio::test]
15868    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15869        let tmp = tempfile::tempdir().expect("tempdir");
15870        let repo = tmp.path().join("repo");
15871        std::fs::create_dir_all(&repo).unwrap();
15872        init_repo(&repo);
15873
15874        let mut config = Config::default();
15875        config.merge.mode = MergeMode::Local;
15876
15877        let mut state = RunState::new(
15878            repo.clone(),
15879            "main".to_owned(),
15880            "deadbeef".to_owned(),
15881            "task".to_owned(),
15882            config,
15883        );
15884        state.candidates = vec![Candidate {
15885            index: 0,
15886            label: 'A',
15887            agent: "alpha".to_owned(),
15888            branch: "does-not-exist".to_owned(),
15889            worktree: repo.clone(),
15890            summary: String::new(),
15891            stat: String::new(),
15892            files: 0,
15893            commits: 0,
15894            empty: false,
15895            failed: None,
15896            verified_noop: None,
15897            duration_ms: 0,
15898            folded: false,
15899        }];
15900        state.tally = Some(Tally {
15901            first_choice: BTreeMap::from([('A', 1)]),
15902            borda: BTreeMap::new(),
15903            winner: 'A',
15904            rankings: 1,
15905            unanimous_initial: true,
15906            deliberated: false,
15907            changed_votes: 0,
15908            unanimous_final: true,
15909            tie_break: None,
15910            judges: 0,
15911            present: 0,
15912            quorum: 0,
15913            met_quorum: true,
15914            uncontested: Some("only candidate A produced a change".to_owned()),
15915        });
15916        state.reviews = vec![ReviewRound {
15917            round: 1,
15918            head: "deadbeef".to_owned(),
15919            verified_head: None,
15920            verified_at: None,
15921            reviews: Vec::new(),
15922            e2e: Vec::new(),
15923            fix: None,
15924            blocking: 0,
15925            answered: 0,
15926            expected: 0,
15927            clean: true,
15928            verify_retried: false,
15929            e2e_deferred: false,
15930            e2e_defer_reason: None,
15931            progressed: false,
15932            vote_split: false,
15933            reconsideration: Vec::new(),
15934            verdict: None,
15935        }];
15936        state.gate = vec![CommandOutcome {
15937            command: "test".to_owned(),
15938            code: Some(0),
15939            output_tail: String::new(),
15940            duration_ms: 0,
15941            resource_blocked: false,
15942        }];
15943        state.gate_ran = true;
15944        // Reached its conclusion already — e.g. `land` closing the PR without
15945        // merging it, which (like the honest `MergeMode::None` path) leaves
15946        // `status` at `Ready`. The recorded outcome is what actually marks
15947        // this node done.
15948        state.status = RunStatus::Ready;
15949        state.merge = Some(MergeOutcome {
15950            mode: MergeMode::Local,
15951            ok: false,
15952            detail: "already concluded".to_owned(),
15953            empty: false,
15954        });
15955
15956        let mut runner = Runner {
15957            state,
15958            roles: ResolvedRoles {
15959                implementers: Vec::new(),
15960                judges: Vec::new(),
15961                reviewers: Vec::new(),
15962                fixer: None,
15963                conductor: conductor(),
15964                implementer_roster: Vec::new(),
15965                judge_roster: Vec::new(),
15966                reviewer_roster: Vec::new(),
15967            },
15968            sem: Arc::new(Semaphore::new(1)),
15969            pause: Pause::new(),
15970            interrupt: Pause::new(),
15971        };
15972
15973        runner.merge().await.expect("merge");
15974
15975        assert_eq!(
15976            runner.state.status,
15977            RunStatus::Ready,
15978            "a concluded run's status must not change on reentry"
15979        );
15980        assert_eq!(
15981            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15982            Some("already concluded"),
15983            "merge must not run again once the node already recorded an outcome"
15984        );
15985    }
15986
15987    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15988    /// when its last attempt was resource-blocked (the shared build cache
15989    /// could not be acquired or confirmed fresh in time - see
15990    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15991    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15992    /// a run merge a tree the gate never actually checked - exactly the case
15993    /// a contended cache produces on every retry until it clears. `merge`
15994    /// must refuse until `gate` has actually recorded an attempt.
15995    #[tokio::test]
15996    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15997        let tmp = tempfile::tempdir().expect("tempdir");
15998        let repo = tmp.path().join("repo");
15999        std::fs::create_dir_all(&repo).unwrap();
16000        init_repo(&repo);
16001
16002        let mut config = Config::default();
16003        config.merge.mode = MergeMode::Local;
16004
16005        let mut state = RunState::new(
16006            repo.clone(),
16007            "main".to_owned(),
16008            "deadbeef".to_owned(),
16009            "task".to_owned(),
16010            config,
16011        );
16012        state.candidates = vec![Candidate {
16013            index: 0,
16014            label: 'A',
16015            agent: "alpha".to_owned(),
16016            branch: "does-not-exist".to_owned(),
16017            worktree: repo.clone(),
16018            summary: String::new(),
16019            stat: String::new(),
16020            files: 0,
16021            commits: 0,
16022            empty: false,
16023            failed: None,
16024            verified_noop: None,
16025            duration_ms: 0,
16026            folded: false,
16027        }];
16028        state.tally = Some(Tally {
16029            first_choice: BTreeMap::from([('A', 1)]),
16030            borda: BTreeMap::new(),
16031            winner: 'A',
16032            rankings: 1,
16033            unanimous_initial: true,
16034            deliberated: false,
16035            changed_votes: 0,
16036            unanimous_final: true,
16037            tie_break: None,
16038            judges: 0,
16039            present: 0,
16040            quorum: 0,
16041            met_quorum: true,
16042            uncontested: Some("only candidate A produced a change".to_owned()),
16043        });
16044        state.reviews = vec![ReviewRound {
16045            round: 1,
16046            head: "deadbeef".to_owned(),
16047            verified_head: None,
16048            verified_at: None,
16049            reviews: Vec::new(),
16050            e2e: Vec::new(),
16051            fix: None,
16052            blocking: 0,
16053            answered: 0,
16054            expected: 0,
16055            clean: true,
16056            verify_retried: false,
16057            e2e_deferred: false,
16058            e2e_defer_reason: None,
16059            progressed: false,
16060            vote_split: false,
16061            reconsideration: Vec::new(),
16062            verdict: None,
16063        }];
16064        // The point: `gate` has not recorded anything yet.
16065        state.gate = Vec::new();
16066        state.gate_ran = false;
16067        state.status = RunStatus::Gating;
16068
16069        let mut runner = Runner {
16070            state,
16071            roles: ResolvedRoles {
16072                implementers: Vec::new(),
16073                judges: Vec::new(),
16074                reviewers: Vec::new(),
16075                fixer: None,
16076                conductor: conductor(),
16077                implementer_roster: Vec::new(),
16078                judge_roster: Vec::new(),
16079                reviewer_roster: Vec::new(),
16080            },
16081            sem: Arc::new(Semaphore::new(1)),
16082            pause: Pause::new(),
16083            interrupt: Pause::new(),
16084        };
16085
16086        runner.merge().await.expect("merge");
16087
16088        assert!(
16089            runner.state.merge.is_none(),
16090            "an empty gate must never be read as a passing one: {:?}",
16091            runner.state.merge
16092        );
16093    }
16094
16095    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
16096    /// commands configured and `merge.mode` is `none` (a review-only run).
16097    /// `gate` must still record a real attempt — zero commands, vacuously
16098    /// passed — rather than leaving `state.gate` empty in a way `merge`
16099    /// cannot tell apart from "never ran"; otherwise the run reaches
16100    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
16101    #[tokio::test]
16102    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16103        let tmp = tempfile::tempdir().expect("tempdir");
16104        let repo = tmp.path().join("repo");
16105        std::fs::create_dir_all(&repo).unwrap();
16106        init_repo(&repo);
16107
16108        // Default config: `verify.gate` empty, `merge.mode` is `none`.
16109        let config = Config::default();
16110
16111        let mut state = RunState::new(
16112            repo.clone(),
16113            "main".to_owned(),
16114            "deadbeef".to_owned(),
16115            "task".to_owned(),
16116            config,
16117        );
16118        state.candidates = vec![Candidate {
16119            index: 0,
16120            label: 'A',
16121            agent: "alpha".to_owned(),
16122            branch: "does-not-exist".to_owned(),
16123            worktree: repo.clone(),
16124            summary: String::new(),
16125            stat: String::new(),
16126            files: 0,
16127            commits: 0,
16128            empty: false,
16129            failed: None,
16130            verified_noop: None,
16131            duration_ms: 0,
16132            folded: false,
16133        }];
16134        state.tally = Some(Tally {
16135            first_choice: BTreeMap::from([('A', 1)]),
16136            borda: BTreeMap::new(),
16137            winner: 'A',
16138            rankings: 1,
16139            unanimous_initial: true,
16140            deliberated: false,
16141            changed_votes: 0,
16142            unanimous_final: true,
16143            tie_break: None,
16144            judges: 0,
16145            present: 0,
16146            quorum: 0,
16147            met_quorum: true,
16148            uncontested: Some("only candidate A produced a change".to_owned()),
16149        });
16150        state.reviews = vec![ReviewRound {
16151            round: 1,
16152            head: "deadbeef".to_owned(),
16153            verified_head: None,
16154            verified_at: None,
16155            reviews: Vec::new(),
16156            e2e: Vec::new(),
16157            fix: None,
16158            blocking: 0,
16159            answered: 0,
16160            expected: 0,
16161            clean: true,
16162            verify_retried: false,
16163            e2e_deferred: false,
16164            e2e_defer_reason: None,
16165            progressed: false,
16166            vote_split: false,
16167            reconsideration: Vec::new(),
16168            verdict: None,
16169        }];
16170
16171        let mut runner = Runner {
16172            state,
16173            roles: ResolvedRoles {
16174                implementers: Vec::new(),
16175                judges: Vec::new(),
16176                reviewers: Vec::new(),
16177                fixer: None,
16178                conductor: conductor(),
16179                implementer_roster: Vec::new(),
16180                judge_roster: Vec::new(),
16181                reviewer_roster: Vec::new(),
16182            },
16183            sem: Arc::new(Semaphore::new(1)),
16184            pause: Pause::new(),
16185            interrupt: Pause::new(),
16186        };
16187
16188        runner.gate().await.expect("gate");
16189        assert!(
16190            runner.state.gate_ran,
16191            "zero configured commands is still a real attempt, not an unrun gate"
16192        );
16193        assert!(runner.state.gate.is_empty());
16194        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16195        assert_ne!(
16196            runner.state.status,
16197            RunStatus::Blocked,
16198            "a gate with nothing to check must not read as failed"
16199        );
16200
16201        runner.merge().await.expect("merge");
16202        assert_eq!(
16203            runner.state.status,
16204            RunStatus::Ready,
16205            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16206        );
16207    }
16208
16209    /// `Config::cache_dir` is derived from `verify.e2e` as well as
16210    /// `verify.gate` (so the e2e leg and the final gate never build against
16211    /// different directories). With zero `verify.gate` commands but a
16212    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
16213    /// that lease before discovering it had nothing to run - so a repo with
16214    /// no gate commands could come back `resource_blocked` (and therefore
16215    /// still `gate_ran == false`) on nothing but an unrelated run holding the
16216    /// cache, exactly the contention this run's own zero commands could
16217    /// never have touched. `gate` must recognise there is nothing to check
16218    /// before it ever asks for the lease.
16219    #[tokio::test]
16220    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16221        crate::run::pin_test_home();
16222        let home = crate::run::home();
16223
16224        let tmp = tempfile::tempdir().expect("tempdir");
16225        let repo = tmp.path().join("repo");
16226        std::fs::create_dir_all(&repo).unwrap();
16227        init_repo(&repo);
16228        // Unique to this test, so holding its lease cannot collide with
16229        // another test sharing the same process-wide `home`.
16230        let cache_dir = tmp.path().join("target");
16231
16232        let mut config = Config::default();
16233        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16234        // `verify.gate` stays empty (the default). Bounded so a regression
16235        // that does start waiting fails the test in seconds, not hangs it.
16236        config.graph.timeout_verify = Some(2);
16237        config.graph.timeout_resource_wait = 2;
16238
16239        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16240        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16241            .expect("no io error acquiring directly")
16242        {
16243            crate::cache::AcquireOutcome::Acquired(g) => g,
16244            crate::cache::AcquireOutcome::Busy(b) => {
16245                panic!("expected the direct acquire to win the lease first: {b:?}")
16246            }
16247        };
16248
16249        let mut state = RunState::new(
16250            repo.clone(),
16251            "main".to_owned(),
16252            "deadbeef".to_owned(),
16253            "task".to_owned(),
16254            config,
16255        );
16256        state.candidates = vec![Candidate {
16257            index: 0,
16258            label: 'A',
16259            agent: "alpha".to_owned(),
16260            branch: "does-not-exist".to_owned(),
16261            worktree: repo.clone(),
16262            summary: String::new(),
16263            stat: String::new(),
16264            files: 0,
16265            commits: 0,
16266            empty: false,
16267            failed: None,
16268            verified_noop: None,
16269            duration_ms: 0,
16270            folded: false,
16271        }];
16272        state.tally = Some(Tally {
16273            first_choice: BTreeMap::from([('A', 1)]),
16274            borda: BTreeMap::new(),
16275            winner: 'A',
16276            rankings: 1,
16277            unanimous_initial: true,
16278            deliberated: false,
16279            changed_votes: 0,
16280            unanimous_final: true,
16281            tie_break: None,
16282            judges: 0,
16283            present: 0,
16284            quorum: 0,
16285            met_quorum: true,
16286            uncontested: Some("only candidate A produced a change".to_owned()),
16287        });
16288        state.reviews = vec![ReviewRound {
16289            round: 1,
16290            head: "deadbeef".to_owned(),
16291            verified_head: None,
16292            verified_at: None,
16293            reviews: Vec::new(),
16294            e2e: Vec::new(),
16295            fix: None,
16296            blocking: 0,
16297            answered: 0,
16298            expected: 0,
16299            clean: true,
16300            verify_retried: false,
16301            e2e_deferred: false,
16302            e2e_defer_reason: None,
16303            progressed: false,
16304            vote_split: false,
16305            reconsideration: Vec::new(),
16306            verdict: None,
16307        }];
16308
16309        let mut runner = Runner {
16310            state,
16311            roles: ResolvedRoles {
16312                implementers: Vec::new(),
16313                judges: Vec::new(),
16314                reviewers: Vec::new(),
16315                fixer: None,
16316                conductor: conductor(),
16317                implementer_roster: Vec::new(),
16318                judge_roster: Vec::new(),
16319                reviewer_roster: Vec::new(),
16320            },
16321            sem: Arc::new(Semaphore::new(1)),
16322            pause: Pause::new(),
16323            interrupt: Pause::new(),
16324        };
16325
16326        let started = std::time::Instant::now();
16327        runner.gate().await.expect("gate");
16328        assert!(
16329            started.elapsed() < Duration::from_secs(1),
16330            "a gate with nothing to run must never wait on a lease it never needed"
16331        );
16332        assert!(
16333            runner.state.gate_ran,
16334            "zero commands is still a real, immediate attempt"
16335        );
16336        assert!(runner.state.gate.is_empty());
16337        assert_ne!(
16338            runner.state.status,
16339            RunStatus::Blocked,
16340            "must not read as resource-blocked on a lease it never asked for"
16341        );
16342    }
16343
16344    /// A separate process carries a live worker's environment throughout all
16345    /// verification entry points; the parallel harness never changes its env.
16346    #[cfg(unix)]
16347    #[tokio::test]
16348    async fn verification_is_standalone_inside_a_live_worker() {
16349        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
16350        if std::env::var_os(CHILD).is_none() {
16351            let home = tempfile::tempdir().unwrap();
16352            let generation = crate::worker::pin_current(home.path()).unwrap();
16353            let output = std::process::Command::new(std::env::current_exe().unwrap())
16354                .args([
16355                    "--exact",
16356                    "graph::tests::verification_is_standalone_inside_a_live_worker",
16357                    "--nocapture",
16358                ])
16359                .env(CHILD, "1")
16360                .env("MAGI_HOME", home.path())
16361                .env("MAGI_WORKER_HOME", home.path())
16362                .env("MAGI_WORKER_ID", "abcdef")
16363                .env("MAGI_WORKER_GENERATION", &generation.id)
16364                .env("MAGI_WORKER_EXE", &generation.executable)
16365                .env("MAGI_RUN", "verification-parent")
16366                .env("MAGI_NODE", "gate")
16367                .env("MAGI_SEAT", "worker-fixture")
16368                .env("MAGI_PROCESS_OWNER", "fixture-owner")
16369                .env("MAGI_NO_AUTOUPDATE", "0")
16370                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
16371                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
16372                .env(
16373                    "VERIFY_EXPECT_TOOLCHAIN",
16374                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
16375                )
16376                .env(
16377                    "VERIFY_EXPECT_CACHE",
16378                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
16379                )
16380                .output()
16381                .unwrap();
16382            assert!(
16383                output.status.success()
16384                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16385                "{}\n{}",
16386                String::from_utf8_lossy(&output.stdout),
16387                String::from_utf8_lossy(&output.stderr)
16388            );
16389            return;
16390        }
16391        let home = crate::run::home();
16392        let directory = home.join("workers/jobs/abcdef");
16393        std::fs::create_dir_all(&directory).unwrap();
16394        let job = crate::worker::Job {
16395            id: "abcdef".into(),
16396            sequence: 1,
16397            kind: "task".into(),
16398            key: "task:verification".into(),
16399            payload: serde_json::json!({}),
16400            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
16401            version: Some(env!("CARGO_PKG_VERSION").into()),
16402            status: crate::worker::Status::Running,
16403            result: None,
16404            pid: Some(std::process::id()),
16405            created_at: 0,
16406            finished_at: None,
16407        };
16408        std::fs::write(
16409            directory.join("job.json"),
16410            serde_json::to_vec(&job).unwrap(),
16411        )
16412        .unwrap();
16413        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
16414        let assert_worker = || {
16415            assert_eq!(
16416                crate::worker::current_job(&home).unwrap().unwrap().id,
16417                job.id
16418            );
16419            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
16420            assert_eq!(
16421                crate::worker::helper_executable(&home).unwrap(),
16422                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
16423            );
16424            assert!(
16425                crate::updater::Checker::new(&crate::config::Update {
16426                    mode: crate::config::UpdateMode::Notify,
16427                    interval: None,
16428                    auto_restart: false,
16429                })
16430                .is_none(),
16431                "the driver remains pinned, even while tests are standalone"
16432            );
16433        };
16434        assert_worker();
16435        let repo = home.join("repo");
16436        std::fs::create_dir_all(&repo).unwrap();
16437        init_repo(&repo);
16438        // Keep storage, seat/control context, containment and tool lookup.
16439        // No inherited values (or credentials) are printed on success.
16440        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
16441            test -z "${MAGI_WORKER_GENERATION+x}" &&
16442            test -z "${MAGI_WORKER_HOME+x}" &&
16443            test -z "${MAGI_WORKER_EXE+x}" &&
16444            test "$MAGI_HOME/repo" -ef . &&
16445            test "$MAGI_RUN" = verification-parent &&
16446            test "$MAGI_NODE" = gate &&
16447            test "$MAGI_SEAT" = worker-fixture &&
16448            test -n "$MAGI_PROCESS_OWNER" &&
16449            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
16450            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
16451            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
16452            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
16453        // Negative control: an ordinary child inherits the contamination.
16454        assert!(
16455            !std::process::Command::new("sh")
16456                .args(["-c", probe])
16457                .current_dir(&repo)
16458                .status()
16459                .unwrap()
16460                .success()
16461        );
16462        let unrelated_home = tempfile::tempdir().unwrap();
16463        assert!(
16464            crate::worker::current_job(unrelated_home.path()).is_err(),
16465            "an inherited worker id cannot be resolved in a unit fixture's home"
16466        );
16467        let mut runner = runner_at(RunStatus::Gating);
16468        runner.state.repo = repo.clone();
16469        runner.state.config.disk.min_free_bytes = 0;
16470        runner.state.config.coordination = crate::config::test_coordination();
16471        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16472        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
16473        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
16474        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
16475        let winner = Candidate {
16476            index: 0,
16477            label: 'A',
16478            agent: "fixture".into(),
16479            branch: "main".into(),
16480            worktree: repo.clone(),
16481            summary: String::new(),
16482            stat: String::new(),
16483            files: 0,
16484            commits: 0,
16485            empty: false,
16486            failed: None,
16487            verified_noop: None,
16488            duration_ms: 0,
16489            folded: false,
16490        };
16491        let (e2e, retried, pids) = run_e2e_with_retry(
16492            &mut runner.state,
16493            &["sh".into(), "-c".into()],
16494            &[format!("{probe} && printf initial")],
16495            &repo,
16496            Duration::from_secs(10),
16497            "initial",
16498        )
16499        .await;
16500        assert!(!retried && pids.is_empty());
16501        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
16502        assert_eq!(e2e[0].output_tail, "initial");
16503        let gate = runner.run_gate(&winner).await.unwrap();
16504        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
16505        // An interrupted repair must recheck e2e from saved state. Landing
16506        // reuses that state and must run both pre-gate and gate again.
16507        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
16508            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
16509            passed: false,
16510            outcomes: Vec::new(),
16511        });
16512        runner.state.save().unwrap();
16513        let mut resumed = RunState::load(&runner.state.id).unwrap();
16514        let wait = crate::repo_lock::WaitPolicy::default();
16515        assert!(matches!(
16516            check_gate_repair_e2e(&mut resumed, &winner, &wait)
16517                .await
16518                .unwrap(),
16519            GateFix::Retry
16520        ));
16521        let saved = RunState::load(&resumed.id).unwrap();
16522        let repair = saved.gate_repair_e2e.unwrap();
16523        assert!(repair.passed);
16524        assert_eq!(repair.outcomes.len(), 1);
16525        assert!(repair.outcomes[0].ok());
16526        let landing = run_landing_gate(&mut resumed, &winner, &wait)
16527            .await
16528            .unwrap();
16529        assert_eq!(landing.len(), 1);
16530        assert!(landing[0].ok(), "{landing:?}");
16531        assert_eq!(
16532            std::fs::read_to_string(repo.join("phases")).unwrap(),
16533            "pre\ngate\ne2e\npre\ngate\n"
16534        );
16535        assert_worker();
16536    }
16537
16538    #[cfg(unix)]
16539    #[tokio::test]
16540    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16541        crate::run::pin_test_home();
16542        let tmp = tempfile::tempdir().unwrap();
16543        let mut config = Config::default();
16544        config.disk.min_free_bytes = 0;
16545        let mut state = RunState::new(
16546            tmp.path().to_owned(),
16547            "main".into(),
16548            "base".into(),
16549            "verification evidence".into(),
16550            config,
16551        );
16552        let (outcomes, pids) = run_commands(
16553            &mut state,
16554            "gate",
16555            "gate",
16556            1,
16557            &["sh".into(), "-c".into()],
16558            &[
16559                "printf before-timeout; exec sleep 30".into(),
16560                "kill -TERM $$".into(),
16561                "exit 7".into(),
16562                "printf success".into(),
16563            ],
16564            tmp.path(),
16565            Duration::from_secs(1),
16566        )
16567        .await;
16568        wait_for_timed_out_children_to_die(&pids).await;
16569        assert_eq!(
16570            pids.len(),
16571            1,
16572            "unleased timeout retains its death-check PID"
16573        );
16574        assert_eq!(outcomes.len(), 4);
16575        assert_eq!(outcomes[0].code, None);
16576        assert!(outcomes[0].output_tail.contains("before-timeout"));
16577        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16578        assert_eq!(outcomes[1].code, None);
16579        assert!(outcomes[1].output_tail.contains("command terminated:"));
16580        assert!(!outcomes[1].output_tail.contains("timed out"));
16581        assert_eq!(outcomes[2].code, Some(7));
16582        assert!(!outcomes[2].output_tail.contains("timed out"));
16583        assert_eq!(outcomes[3].code, Some(0));
16584        assert_eq!(outcomes[3].output_tail, "success");
16585        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16586    }
16587
16588    /// The addendum's second gap: a `verify.gate` command running for real
16589    /// wall-clock time had nothing at all to show for it in `active` before
16590    /// `run_commands` learned to record it — a run could sit in `Gating` for
16591    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
16592    /// what was actually happening. Proven with a genuinely still-running
16593    /// command, not just a before/after check on the final state: a poller
16594    /// task reads the same `run.json` `gate()` is writing, the same way the
16595    /// phone or `magi show` would, while the shell command is still blocked
16596    /// on its own release marker.
16597    #[tokio::test]
16598    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16599        crate::run::pin_test_home();
16600
16601        let tmp = tempfile::tempdir().expect("tempdir");
16602        let repo = tmp.path().join("repo");
16603        std::fs::create_dir_all(&repo).unwrap();
16604        init_repo(&repo);
16605
16606        let mut config = Config::default();
16607        config.verify.gate = vec![
16608            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16609             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16610                .to_owned(),
16611        ];
16612
16613        let mut state = RunState::new(
16614            repo.clone(),
16615            "main".to_owned(),
16616            "deadbeef".to_owned(),
16617            "task".to_owned(),
16618            config,
16619        );
16620        let run_id = state.id.clone();
16621        state.candidates = vec![Candidate {
16622            index: 0,
16623            label: 'A',
16624            agent: "alpha".to_owned(),
16625            branch: "does-not-exist".to_owned(),
16626            worktree: repo.clone(),
16627            summary: String::new(),
16628            stat: String::new(),
16629            files: 0,
16630            commits: 0,
16631            empty: false,
16632            failed: None,
16633            verified_noop: None,
16634            duration_ms: 0,
16635            folded: false,
16636        }];
16637        state.tally = Some(Tally {
16638            first_choice: BTreeMap::from([('A', 1)]),
16639            borda: BTreeMap::new(),
16640            winner: 'A',
16641            rankings: 1,
16642            unanimous_initial: true,
16643            deliberated: false,
16644            changed_votes: 0,
16645            unanimous_final: true,
16646            tie_break: None,
16647            judges: 0,
16648            present: 0,
16649            quorum: 0,
16650            met_quorum: true,
16651            uncontested: Some("only candidate A produced a change".to_owned()),
16652        });
16653        state.reviews = vec![ReviewRound {
16654            round: 1,
16655            head: "deadbeef".to_owned(),
16656            verified_head: None,
16657            verified_at: None,
16658            reviews: Vec::new(),
16659            e2e: Vec::new(),
16660            fix: None,
16661            blocking: 0,
16662            answered: 0,
16663            expected: 0,
16664            clean: true,
16665            verify_retried: false,
16666            e2e_deferred: false,
16667            e2e_defer_reason: None,
16668            progressed: false,
16669            vote_split: false,
16670            reconsideration: Vec::new(),
16671            verdict: None,
16672        }];
16673
16674        let mut runner = Runner {
16675            state,
16676            roles: ResolvedRoles {
16677                implementers: Vec::new(),
16678                judges: Vec::new(),
16679                reviewers: Vec::new(),
16680                fixer: None,
16681                conductor: conductor(),
16682                implementer_roster: Vec::new(),
16683                judge_roster: Vec::new(),
16684                reviewer_roster: Vec::new(),
16685            },
16686            sem: Arc::new(Semaphore::new(1)),
16687            pause: Pause::new(),
16688            interrupt: Pause::new(),
16689        };
16690
16691        let started_marker = repo.join("started.marker");
16692        let release_marker = repo.join("release.marker");
16693        let poller = tokio::spawn(async move {
16694            // Bounded so a regression that never records the task entry
16695            // fails this test in seconds instead of hanging the suite —
16696            // the same shape `a_park_requested_while_a_seat_is_mid_call_
16697            // does_not_cut_it_short` uses for the same reason.
16698            for _ in 0..100 {
16699                if started_marker.exists()
16700                    && let Ok(s) = crate::run::RunState::load(&run_id)
16701                    && let Some(a) = s.active.get("gate")
16702                {
16703                    std::fs::write(&release_marker, b"go").expect("release marker");
16704                    return Some(a.clone());
16705                }
16706                tokio::time::sleep(Duration::from_millis(50)).await;
16707            }
16708            None
16709        });
16710
16711        runner.gate().await.expect("gate");
16712        let captured = poller.await.expect("poller task");
16713        let captured = captured.expect(
16714            "the poller never saw a `gate` task entry in run.json while the command was \
16715             still blocked on its own release marker",
16716        );
16717
16718        assert_eq!(captured.task.as_deref(), Some("gate"));
16719        assert_eq!(captured.node, "gate");
16720        assert_eq!(captured.index, Some(1));
16721        assert_eq!(captured.total, Some(1));
16722        assert!(
16723            captured
16724                .command
16725                .as_deref()
16726                .is_some_and(|c| c.contains("started.marker")),
16727            "{captured:?}"
16728        );
16729
16730        assert!(
16731            runner.state.active.is_empty(),
16732            "the entry must be cleared once the command actually finished: {:?}",
16733            runner.state.active
16734        );
16735        assert!(runner.state.gate_ran);
16736        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16737    }
16738
16739    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16740    /// record for `land`; one with only a Minor, or no reject, leaves none.
16741    #[tokio::test]
16742    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16743        use crate::verdict::{Finding, ReviewVote, Severity};
16744        crate::run::pin_test_home();
16745        let tmp = tempfile::tempdir().expect("tempdir");
16746        let repo = tmp.path().join("repo");
16747        std::fs::create_dir_all(&repo).unwrap();
16748        init_repo(&repo);
16749
16750        for (severity, vote, expect) in [
16751            (Severity::Major, ReviewVote::Reject, true),
16752            (Severity::Minor, ReviewVote::Reject, false),
16753            (Severity::Major, ReviewVote::Approve, false),
16754        ] {
16755            let mut round = review_round(false, 1, 1, 1, false, true);
16756            round.reviews = vec![ReviewRecord {
16757                reviewer: 1,
16758                agent: "alpha".to_owned(),
16759                summary: String::new(),
16760                findings: vec![Finding {
16761                    id: "R1-1-1".to_owned(),
16762                    severity,
16763                    file: None,
16764                    line: None,
16765                    title: "t".to_owned(),
16766                    detail: String::new(),
16767                }],
16768                vote: Some(vote),
16769                failed: None,
16770                duration_ms: 0,
16771                attempts: 0,
16772            }];
16773            let mut state = RunState::new(
16774                repo.clone(),
16775                "main".to_owned(),
16776                "deadbeef".to_owned(),
16777                "task".to_owned(),
16778                Config::default(),
16779            );
16780            state.reviews = vec![round];
16781            let mut runner = Runner {
16782                state,
16783                roles: ResolvedRoles {
16784                    implementers: Vec::new(),
16785                    judges: Vec::new(),
16786                    reviewers: Vec::new(),
16787                    fixer: None,
16788                    conductor: conductor(),
16789                    implementer_roster: Vec::new(),
16790                    judge_roster: Vec::new(),
16791                    reviewer_roster: Vec::new(),
16792                },
16793                sem: Arc::new(Semaphore::new(1)),
16794                pause: Pause::new(),
16795                interrupt: Pause::new(),
16796            };
16797            let shell = runner.state.config.shell();
16798            runner
16799                .stop_reviewing("round budget spent", &shell, &repo)
16800                .await
16801                .expect("stop_reviewing");
16802            assert_eq!(runner.state.status, RunStatus::Gating);
16803            assert_eq!(
16804                runner.state.contested_handoff.is_some(),
16805                expect,
16806                "{severity:?} + {vote:?}"
16807            );
16808        }
16809    }
16810
16811    /// The shape the incident this whole fix responds to actually had: the
16812    /// round budget spent, the last round's own e2e blocked on the shared
16813    /// build cache (held here by a live pid — this test process — exactly
16814    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16815    /// without forking a process). `stop_reviewing` must retry it — not
16816    /// silently leave the round looking untouched (the catch-up-only half of
16817    /// the bug), and not read the contention as a red `e2e` and block the
16818    /// run on it (the other half). Called directly, the same way
16819    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16820    /// above exercises `gate`, so this never needs a real cargo build to
16821    /// reach: the lease is never released, so `with_cache_lease` never gets
16822    /// past acquiring it into anything that would need a real workspace.
16823    #[tokio::test]
16824    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16825        crate::run::pin_test_home();
16826        let home = crate::run::home();
16827
16828        let tmp = tempfile::tempdir().expect("tempdir");
16829        let repo = tmp.path().join("repo");
16830        std::fs::create_dir_all(&repo).unwrap();
16831        init_repo(&repo);
16832        let head = crate::git::rev_parse(&repo, "HEAD")
16833            .await
16834            .expect("rev-parse");
16835        // Unique to this test, so holding its lease cannot collide with
16836        // another test sharing the same process-wide `home`.
16837        let cache_dir = tmp.path().join("target");
16838
16839        let mut config = Config::default();
16840        config.verify.e2e = vec![format!(
16841            "CARGO_TARGET_DIR='{}' test -f README.md",
16842            cache_dir.display()
16843        )];
16844        config.graph.review_rounds = 1;
16845        // Bounded so a regression that does start waiting fails the test in
16846        // seconds, not hangs it.
16847        config.graph.timeout_verify = Some(2);
16848        config.graph.timeout_resource_wait = 2;
16849
16850        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16851        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16852            .expect("no io error acquiring directly")
16853        {
16854            crate::cache::AcquireOutcome::Acquired(g) => g,
16855            crate::cache::AcquireOutcome::Busy(b) => {
16856                panic!("expected the direct acquire to win the lease first: {b:?}")
16857            }
16858        };
16859
16860        let mut state = RunState::new(
16861            repo.clone(),
16862            "main".to_owned(),
16863            head.clone(),
16864            "task".to_owned(),
16865            config,
16866        );
16867        state.candidates = vec![Candidate {
16868            index: 0,
16869            label: 'A',
16870            agent: "alpha".to_owned(),
16871            branch: "does-not-exist".to_owned(),
16872            worktree: repo.clone(),
16873            summary: String::new(),
16874            stat: String::new(),
16875            files: 0,
16876            commits: 0,
16877            empty: false,
16878            failed: None,
16879            verified_noop: None,
16880            duration_ms: 0,
16881            folded: false,
16882        }];
16883        state.tally = Some(Tally {
16884            first_choice: BTreeMap::from([('A', 1)]),
16885            borda: BTreeMap::new(),
16886            winner: 'A',
16887            rankings: 1,
16888            unanimous_initial: true,
16889            deliberated: false,
16890            changed_votes: 0,
16891            unanimous_final: true,
16892            tie_break: None,
16893            judges: 0,
16894            present: 0,
16895            quorum: 0,
16896            met_quorum: true,
16897            uncontested: Some("only candidate A produced a change".to_owned()),
16898        });
16899        // The round budget's last round, deferred: `needs_catchup_run`'s
16900        // other trigger. `stop_reviewing`'s retry machinery must treat this
16901        // exactly like a resource-blocked attempt once it actually runs.
16902        state.reviews = vec![ReviewRound {
16903            round: 1,
16904            head: head.clone(),
16905            verified_head: None,
16906            verified_at: None,
16907            reviews: Vec::new(),
16908            e2e: Vec::new(),
16909            fix: None,
16910            blocking: 1,
16911            answered: 1,
16912            expected: 1,
16913            clean: false,
16914            verify_retried: false,
16915            e2e_deferred: true,
16916            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16917            progressed: false,
16918            vote_split: false,
16919            reconsideration: Vec::new(),
16920            verdict: None,
16921        }];
16922
16923        let mut runner = Runner {
16924            state,
16925            roles: ResolvedRoles {
16926                implementers: Vec::new(),
16927                judges: Vec::new(),
16928                reviewers: Vec::new(),
16929                fixer: None,
16930                conductor: conductor(),
16931                implementer_roster: Vec::new(),
16932                judge_roster: Vec::new(),
16933                reviewer_roster: Vec::new(),
16934            },
16935            sem: Arc::new(Semaphore::new(1)),
16936            pause: Pause::new(),
16937            interrupt: Pause::new(),
16938        };
16939
16940        let shell = runner.state.config.shell();
16941        runner
16942            .stop_reviewing("round budget spent", &shell, &repo)
16943            .await
16944            .expect("stop_reviewing");
16945
16946        let last = runner.state.reviews.last().expect("round record");
16947        assert_eq!(
16948            last.e2e_status(),
16949            E2eStatus::ResourceBlocked,
16950            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16951             failed: {last:?}"
16952        );
16953        assert_eq!(
16954            last.verified_head.as_deref(),
16955            Some(head.as_str()),
16956            "which commit this attempt targeted is known even though nothing finished checking \
16957             it"
16958        );
16959        let first_attempt_at = last
16960            .verified_at
16961            .expect("when this attempt ran is known too");
16962        assert_ne!(
16963            runner.state.status,
16964            RunStatus::Blocked,
16965            "contention is evidence about the machine, not the patch — it must not settle the \
16966             run as blocked: {:?}",
16967            runner.state.status
16968        );
16969        assert!(
16970            !runner
16971                .state
16972                .events
16973                .iter()
16974                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16975            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16976            runner.state.events
16977        );
16978
16979        // The cache is still held: a later reentry must retry the same
16980        // round's verification again — not leave it looking exactly as
16981        // untouched as the first blocked attempt, which is indistinguishable
16982        // from never having tried again at all.
16983        runner
16984            .stop_reviewing("round budget spent", &shell, &repo)
16985            .await
16986            .expect("stop_reviewing retry");
16987        assert_eq!(
16988            runner.state.reviews.len(),
16989            1,
16990            "no new round was started: {:?}",
16991            runner.state.reviews
16992        );
16993        let last = runner.state.reviews.last().expect("round record");
16994        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16995        assert!(
16996            last.verified_at.expect("still known") > first_attempt_at,
16997            "a second reentry must be a fresh attempt, not a stale copy of the first"
16998        );
16999        assert_ne!(runner.state.status, RunStatus::Blocked);
17000
17001        held.release();
17002    }
17003
17004    #[cfg(unix)]
17005    #[tokio::test]
17006    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
17007        ask_test_home();
17008        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
17009            let tmp = tempfile::tempdir().unwrap();
17010            let repo = tmp.path().join("repo");
17011            std::fs::create_dir(&repo).unwrap();
17012            init_repo(&repo);
17013            let run_git = |args: &[&str]| {
17014                let out = std::process::Command::new("git")
17015                    .args(args)
17016                    .current_dir(&repo)
17017                    .output()
17018                    .unwrap();
17019                assert!(
17020                    out.status.success(),
17021                    "{args:?}: {}",
17022                    String::from_utf8_lossy(&out.stderr)
17023                );
17024            };
17025            let origin = tmp.path().join("origin.git");
17026            run_git(&["init", "--bare", origin.to_str().unwrap()]);
17027            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17028            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17029            run_git(&["add", "spec.rs"]);
17030            run_git(&["commit", "-m", "initial spec"]);
17031            run_git(&["push", "origin", "main"]);
17032            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17033            run_git(&["checkout", "-b", "fix/review"]);
17034            std::fs::write(
17035                repo.join("check.rs"),
17036                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17037            )
17038            .unwrap();
17039            run_git(&["add", "check.rs"]);
17040            run_git(&["commit", "-m", "use spec"]);
17041            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17042            run_git(&["checkout", "main"]);
17043            std::fs::write(
17044                repo.join("spec.rs"),
17045                "struct AgentSpec { max_turn_secs: u64 }\n",
17046            )
17047            .unwrap();
17048            run_git(&["add", "spec.rs"]);
17049            run_git(&["commit", "-m", "extend spec"]);
17050            run_git(&["push", "origin", "main"]);
17051            run_git(&["checkout", "fix/review"]);
17052
17053            let mut runner = runner_at(RunStatus::Reviewing);
17054            runner.state.repo = repo.clone();
17055            runner.state.base_commit = base;
17056            runner.state.pr_head = Some("fix/review".into());
17057            runner.state.config.graph.review_rounds = max_rounds;
17058            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17059            runner.state.config.graph.sessions = false;
17060            runner.state.config.disk.min_free_bytes = 0;
17061            runner.state.config.verify.e2e =
17062                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17063            runner.state.candidates = vec![Candidate {
17064                index: 0,
17065                label: 'A',
17066                agent: "fixture".into(),
17067                branch: "fix/review".into(),
17068                worktree: repo.clone(),
17069                summary: String::new(),
17070                stat: String::new(),
17071                files: 1,
17072                commits: 1,
17073                empty: false,
17074                failed: None,
17075                verified_noop: None,
17076                duration_ms: 0,
17077                folded: false,
17078            }];
17079            runner.state.tally = Some(Tally {
17080                first_choice: BTreeMap::new(),
17081                borda: BTreeMap::new(),
17082                winner: 'A',
17083                rankings: 0,
17084                unanimous_initial: true,
17085                deliberated: false,
17086                changed_votes: 0,
17087                unanimous_final: true,
17088                tie_break: None,
17089                judges: 0,
17090                present: 0,
17091                quorum: 0,
17092                met_quorum: true,
17093                uncontested: Some("review-only".into()),
17094            });
17095            let mut last = review_round(false, 0, 1, 1, false, false);
17096            last.head = reviewed.clone();
17097            last.e2e_deferred = true;
17098            last.e2e[0].resource_blocked = true;
17099            last.e2e[0].output_tail = "waiting for the shared build cache".into();
17100            if max_rounds == 1 {
17101                last.e2e.clear();
17102            }
17103            runner.state.reviews = vec![last];
17104            let calls = tmp.path().join("fix-calls");
17105            let mut fixer = spec("repair");
17106            fixer
17107                .env
17108                .insert("FIX_CALLS".into(), calls.display().to_string());
17109            fixer.command = vec!["sh".into(), "-c".into(), r#"
17110                echo fix >> "$FIX_CALLS"
17111                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17112                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17113                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17114                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17115            "#.into()];
17116            let mut reviewer = spec("reviewer");
17117            reviewer.command = vec![
17118                "sh".into(),
17119                "-c".into(),
17120                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17121                    .into(),
17122            ];
17123            runner.roles.fixer = Some(vec![fixer]);
17124            runner.roles.reviewers = vec![reviewer];
17125
17126            runner.sync_to_base().await.unwrap();
17127            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17128            assert_ne!(rebased, reviewed);
17129            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17130            if reentry {
17131                // Simulate a crash after catch-up evidence was saved but before
17132                // dispatching the fixer. No dedicated pending-repair field exists.
17133                let shell = runner.state.config.shell();
17134                runner
17135                    .stop_unresolved_review("deferred verification", &shell, &repo)
17136                    .await
17137                    .unwrap();
17138                assert!(runner.catchup_repairable());
17139                assert!(!calls.exists());
17140                runner.state = RunState::load(&runner.state.id).unwrap();
17141                assert!(runner.catchup_repairable());
17142            }
17143            runner.review_loop().await.unwrap();
17144            let first = &runner.state.reviews[0];
17145            assert_eq!(first.head, reviewed);
17146            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17147            assert!(
17148                first.verify_retried,
17149                "a reproduced build failure gets exactly one retry: {:?}",
17150                first.e2e
17151            );
17152            assert!(first.e2e[0].output_tail.contains("E0063"));
17153            assert!(!runner.catchup_repairable());
17154            if max_rounds == 2 {
17155                assert_eq!(
17156                    runner.state.status,
17157                    RunStatus::Gating,
17158                    "{}",
17159                    crate::report::run(&runner.state)
17160                );
17161                assert_eq!(runner.state.reviews.len(), 2);
17162                assert!(first.fix.as_ref().unwrap().committed);
17163                assert!(runner.state.reviews[1].clean);
17164                assert_eq!(
17165                    runner
17166                        .state
17167                        .reviews
17168                        .iter()
17169                        .filter(|r| r.fix.is_some())
17170                        .count(),
17171                    1
17172                );
17173                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17174                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17175            } else {
17176                assert_eq!(runner.state.status, RunStatus::Blocked);
17177                assert!(first.fix.is_none());
17178                assert!(!calls.exists());
17179                let recovery = crate::recovery::assess(
17180                    "prior",
17181                    Some(&runner.state),
17182                    &runner.landing_base(),
17183                    crate::recovery::Relation::Diverged,
17184                );
17185                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17186                let reasons = recovery.reasons.join("\n");
17187                assert!(
17188                    reasons.contains(&reviewed) && reasons.contains(&rebased),
17189                    "{reasons}"
17190                );
17191                assert!(
17192                    reasons.contains("base sync to")
17193                        && reasons.contains("additional review is required"),
17194                    "{reasons}"
17195                );
17196            }
17197        }
17198    }
17199
17200    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
17201    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
17202    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
17203    /// own top-of-function fast path (`review_conclusion`) correctly reads
17204    /// this shape as `None` rather than guessing `Blocked`, and the loop's
17205    /// own `for` range is empty once the round budget is spent, so
17206    /// `review_loop` must retry the check itself rather than silently doing
17207    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
17208    /// above exercises directly, but through `review_loop`'s own entry point
17209    /// this time, proving the wiring between the two rather than just the
17210    /// retry logic in isolation.
17211    #[tokio::test]
17212    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17213        crate::run::pin_test_home();
17214        let home = crate::run::home();
17215
17216        let tmp = tempfile::tempdir().expect("tempdir");
17217        let repo = tmp.path().join("repo");
17218        std::fs::create_dir_all(&repo).unwrap();
17219        init_repo(&repo);
17220        let head = crate::git::rev_parse(&repo, "HEAD")
17221            .await
17222            .expect("rev-parse");
17223        let cache_dir = tmp.path().join("target");
17224
17225        let mut config = Config::default();
17226        config.verify.e2e = vec![format!(
17227            "CARGO_TARGET_DIR='{}' test -f README.md",
17228            cache_dir.display()
17229        )];
17230        config.graph.review_rounds = 1;
17231        config.graph.timeout_verify = Some(2);
17232        config.graph.timeout_resource_wait = 2;
17233
17234        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17235        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17236            .expect("no io error acquiring directly")
17237        {
17238            crate::cache::AcquireOutcome::Acquired(g) => g,
17239            crate::cache::AcquireOutcome::Busy(b) => {
17240                panic!("expected the direct acquire to win the lease first: {b:?}")
17241            }
17242        };
17243
17244        let mut state = RunState::new(
17245            repo.clone(),
17246            "main".to_owned(),
17247            head.clone(),
17248            "task".to_owned(),
17249            config,
17250        );
17251        state.candidates = vec![Candidate {
17252            index: 0,
17253            label: 'A',
17254            agent: "alpha".to_owned(),
17255            branch: "does-not-exist".to_owned(),
17256            worktree: repo.clone(),
17257            summary: String::new(),
17258            stat: String::new(),
17259            files: 0,
17260            commits: 0,
17261            empty: false,
17262            failed: None,
17263            verified_noop: None,
17264            duration_ms: 0,
17265            folded: false,
17266        }];
17267        state.tally = Some(Tally {
17268            first_choice: BTreeMap::from([('A', 1)]),
17269            borda: BTreeMap::new(),
17270            winner: 'A',
17271            rankings: 1,
17272            unanimous_initial: true,
17273            deliberated: false,
17274            changed_votes: 0,
17275            unanimous_final: true,
17276            tie_break: None,
17277            judges: 0,
17278            present: 0,
17279            quorum: 0,
17280            met_quorum: true,
17281            uncontested: Some("only candidate A produced a change".to_owned()),
17282        });
17283        // The exact shape a prior process's `stop_reviewing` would have left
17284        // on disk: the round budget's last round, a real attempt already
17285        // made and already resource-blocked.
17286        state.reviews = vec![ReviewRound {
17287            round: 1,
17288            head: head.clone(),
17289            verified_head: Some(head.clone()),
17290            verified_at: Some(jiff::Timestamp::now()),
17291            reviews: Vec::new(),
17292            e2e: vec![CommandOutcome {
17293                command: format!(
17294                    "CARGO_TARGET_DIR='{}' test -f README.md",
17295                    cache_dir.display()
17296                ),
17297                code: None,
17298                output_tail: "waiting for the shared build cache".to_owned(),
17299                duration_ms: 0,
17300                resource_blocked: true,
17301            }],
17302            fix: None,
17303            blocking: 1,
17304            answered: 1,
17305            expected: 1,
17306            clean: false,
17307            verify_retried: false,
17308            e2e_deferred: false,
17309            e2e_defer_reason: None,
17310            progressed: false,
17311            vote_split: false,
17312            reconsideration: Vec::new(),
17313            verdict: None,
17314        }];
17315
17316        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17317        let mut runner = Runner {
17318            state,
17319            roles: ResolvedRoles {
17320                implementers: Vec::new(),
17321                judges: Vec::new(),
17322                reviewers: Vec::new(),
17323                fixer: None,
17324                conductor: conductor(),
17325                implementer_roster: Vec::new(),
17326                judge_roster: Vec::new(),
17327                reviewer_roster: Vec::new(),
17328            },
17329            sem: Arc::new(Semaphore::new(1)),
17330            pause: Pause::new(),
17331            interrupt: Pause::new(),
17332        };
17333
17334        // The lease is still held throughout, so this reentry's own retry is
17335        // also contended — proving `review_loop` actually tried again (not
17336        // that it happened to succeed) is what the timestamp comparison
17337        // below is for.
17338        runner.review_loop().await.expect("review_loop");
17339
17340        assert_eq!(
17341            runner.state.reviews.len(),
17342            1,
17343            "no new round was started on top of the unresolved one: {:?}",
17344            runner.state.reviews
17345        );
17346        let last = &runner.state.reviews[0];
17347        assert_eq!(
17348            last.e2e_status(),
17349            E2eStatus::ResourceBlocked,
17350            "still contended: {last:?}"
17351        );
17352        assert!(
17353            last.verified_at.expect("still known") > first_attempt_at,
17354            "review_loop must have actually retried the check, not left it exactly as found"
17355        );
17356        assert_ne!(
17357            runner.state.status,
17358            RunStatus::Blocked,
17359            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17360            runner.state.status
17361        );
17362
17363        held.release();
17364    }
17365
17366    #[tokio::test]
17367    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17368        crate::run::pin_test_home();
17369        for status in [RunStatus::Blocked, RunStatus::Gating] {
17370            let mut runner = runner_at(status);
17371            runner.state.config.graph.land = true;
17372            runner.state.gate_ran = true;
17373            runner.state.merge = Some(MergeOutcome {
17374                mode: MergeMode::Pr,
17375                ok: false,
17376                detail: "the fixer produced no commit".into(),
17377                empty: false,
17378            });
17379            let url = "https://example.invalid/o/r/pull/1";
17380            runner.state.pr = Some(crate::run::PrRecord {
17381                url: url.into(),
17382                number: 1,
17383                state: "open".into(),
17384                checks: "red".into(),
17385                round: 1,
17386                rounds: 1,
17387                red_at_merge: Vec::new(),
17388            });
17389            runner.state.save().unwrap();
17390            // The nonexistent repository makes observation fail before a forge
17391            // command can run. We still must reach land, rather than silently
17392            // return from the graph with its old gate and merge records.
17393            runner.execute_graph().await.unwrap();
17394            assert_eq!(runner.state.status, RunStatus::Blocked);
17395            assert!(runner.state.land_sync_pending);
17396            assert!(runner.state.events.iter().any(|event|
17397                event.node == "land" && event.message == format!("watching {url}")));
17398            assert_eq!(
17399                runner.state.merge.as_ref().unwrap().detail,
17400                "the fixer produced no commit"
17401            );
17402        }
17403    }
17404
17405    #[tokio::test]
17406    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17407        crate::run::pin_test_home();
17408        let tmp = tempfile::tempdir().expect("tempdir");
17409        let repo = tmp.path().join("repo");
17410        std::fs::create_dir_all(&repo).unwrap();
17411        init_repo(&repo);
17412
17413        let mut config = Config::default();
17414        config.merge.mode = MergeMode::Pr;
17415        config.graph.land = true;
17416        config.graph.land_approval = false;
17417
17418        let mut state = RunState::new(
17419            repo.clone(),
17420            "main".to_owned(),
17421            "deadbeef".to_owned(),
17422            "task".to_owned(),
17423            config,
17424        );
17425        state.candidates = vec![Candidate {
17426            index: 0,
17427            label: 'A',
17428            agent: "alpha".to_owned(),
17429            branch: "does-not-exist".to_owned(),
17430            worktree: repo.clone(),
17431            summary: String::new(),
17432            stat: String::new(),
17433            files: 0,
17434            commits: 0,
17435            empty: false,
17436            failed: None,
17437            verified_noop: None,
17438            duration_ms: 0,
17439            folded: false,
17440        }];
17441        state.tally = Some(Tally {
17442            first_choice: BTreeMap::from([('A', 1)]),
17443            borda: BTreeMap::new(),
17444            winner: 'A',
17445            rankings: 1,
17446            unanimous_initial: true,
17447            deliberated: false,
17448            changed_votes: 0,
17449            unanimous_final: true,
17450            tie_break: None,
17451            judges: 0,
17452            present: 0,
17453            quorum: 0,
17454            met_quorum: true,
17455            uncontested: Some("only candidate A produced a change".to_owned()),
17456        });
17457        state.reviews = vec![ReviewRound {
17458            round: 1,
17459            head: "deadbeef".to_owned(),
17460            verified_head: None,
17461            verified_at: None,
17462            reviews: Vec::new(),
17463            e2e: Vec::new(),
17464            fix: None,
17465            blocking: 0,
17466            answered: 0,
17467            expected: 0,
17468            clean: true,
17469            verify_retried: false,
17470            e2e_deferred: false,
17471            e2e_defer_reason: None,
17472            progressed: false,
17473            vote_split: false,
17474            reconsideration: Vec::new(),
17475            verdict: None,
17476        }];
17477        state.gate = vec![CommandOutcome {
17478            command: "test".to_owned(),
17479            code: Some(0),
17480            output_tail: String::new(),
17481            duration_ms: 0,
17482            resource_blocked: false,
17483        }];
17484        state.gate_ran = true;
17485        // A first pass through `merge` already pushed and opened this pull
17486        // request; `status` is `Landing` because a previous call into `land`
17487        // parked or was interrupted before it reached a terminal outcome.
17488        state.status = RunStatus::Landing;
17489        state.merge = Some(MergeOutcome {
17490            mode: MergeMode::Pr,
17491            ok: true,
17492            detail: "https://example.invalid/x/y/pull/1".to_owned(),
17493            empty: false,
17494        });
17495
17496        // The Landing-resume shortcut calls `run_land` directly rather than
17497        // through `merge`, which is exactly the call site that used to skip
17498        // `settle_questions` - see the fixture below.
17499        ask_test_home();
17500        // A real resume loads a persisted run; driver ownership is checked
17501        // against that record before the Landing shortcut is entered.
17502        state.save().expect("save landing run");
17503        let store = ask::Questions::open();
17504        let q = ask_open_question(&store, &state.id);
17505
17506        let mut runner = Runner {
17507            state,
17508            roles: ResolvedRoles {
17509                implementers: Vec::new(),
17510                judges: Vec::new(),
17511                reviewers: Vec::new(),
17512                fixer: None,
17513                conductor: conductor(),
17514                implementer_roster: Vec::new(),
17515                judge_roster: Vec::new(),
17516                reviewer_roster: Vec::new(),
17517            },
17518            sem: Arc::new(Semaphore::new(1)),
17519            pause: Pause::new(),
17520            interrupt: Pause::new(),
17521        };
17522
17523        // `execute`, not `merge` directly: the Landing-resume shortcut lives
17524        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
17525        // exactly because `review_loop` would otherwise clobber the marker
17526        // first.
17527        runner.execute().await.expect("execute");
17528
17529        assert_eq!(
17530            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17531            Some("https://example.invalid/x/y/pull/1"),
17532            "reentry must not push again or open a second pull request over the \
17533             one `land` is already watching"
17534        );
17535        assert_ne!(
17536            runner.state.status,
17537            RunStatus::Landing,
17538            "land could not actually reach the fake pull request, so it must \
17539             have given up rather than left the run silently parked forever"
17540        );
17541        // `land` could not reach the fake pull request, so it gave up into
17542        // `Blocked` - still resumable, so the question must not have been
17543        // swept just because this branch now also calls `settle_questions`.
17544        assert_eq!(runner.state.status, RunStatus::Blocked);
17545        assert!(
17546            store.get(&q.id).unwrap().status.open(),
17547            "Blocked is still alive; settle_questions must have been a no-op here"
17548        );
17549    }
17550
17551    fn state_with_round(round: ReviewRound) -> RunState {
17552        let mut s = RunState::new(
17553            PathBuf::from("/repo"),
17554            "main".to_owned(),
17555            "abc1234".to_owned(),
17556            "add retries".to_owned(),
17557            Config::default(),
17558        );
17559        s.reviews = vec![round];
17560        s
17561    }
17562
17563    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17564        crate::verdict::Finding {
17565            id: id.to_owned(),
17566            severity,
17567            file: None,
17568            line: None,
17569            title: title.to_owned(),
17570            detail: String::new(),
17571        }
17572    }
17573
17574    #[test]
17575    fn pr_body_names_open_findings_and_declined_ones() {
17576        let round = ReviewRound {
17577            round: 2,
17578            head: "deadbee".to_owned(),
17579            verified_head: None,
17580            verified_at: None,
17581            reviews: vec![ReviewRecord {
17582                attempts: 0,
17583                reviewer: 1,
17584                agent: "alpha".to_owned(),
17585                summary: String::new(),
17586                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17587                vote: None,
17588                failed: None,
17589                duration_ms: 0,
17590            }],
17591            e2e: vec![CommandOutcome {
17592                command: "cargo test".to_owned(),
17593                code: Some(0),
17594                output_tail: String::new(),
17595                duration_ms: 0,
17596                resource_blocked: false,
17597            }],
17598            verify_retried: false,
17599            e2e_deferred: false,
17600            e2e_defer_reason: None,
17601            fix: Some(FixRecord {
17602                agent: "alpha".to_owned(),
17603                addressed: Vec::new(),
17604                rejected: vec![crate::verdict::Rejection {
17605                    id: "R1-1-1".to_owned(),
17606                    why: "not reachable from any caller".to_owned(),
17607                }],
17608                notes: String::new(),
17609                committed: true,
17610                failed: None,
17611                duration_ms: 0,
17612                continuation: None,
17613            }),
17614            blocking: 0,
17615            answered: 1,
17616            expected: 1,
17617            clean: false,
17618            progressed: true,
17619            vote_split: false,
17620            reconsideration: Vec::new(),
17621            verdict: None,
17622        };
17623        let state = state_with_round(round);
17624        let body = pr_message(&state, 'A').body;
17625
17626        assert!(body.contains("add retries"), "the task must still be there");
17627        assert!(body.contains("R2-1-1"), "{body}");
17628        assert!(!body.contains("unused import"), "{body}");
17629        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17630        assert!(
17631            !body.contains("not reachable from any caller"),
17632            "the reason it was declined: {body}"
17633        );
17634    }
17635
17636    #[test]
17637    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17638        let round = ReviewRound {
17639            round: 1,
17640            head: "deadbee".to_owned(),
17641            verified_head: None,
17642            verified_at: None,
17643            reviews: vec![ReviewRecord {
17644                attempts: 0,
17645                reviewer: 1,
17646                agent: "alpha".to_owned(),
17647                summary: String::new(),
17648                findings: Vec::new(),
17649                vote: None,
17650                failed: None,
17651                duration_ms: 0,
17652            }],
17653            e2e: Vec::new(),
17654            verify_retried: false,
17655            e2e_deferred: false,
17656            e2e_defer_reason: None,
17657            fix: None,
17658            blocking: 0,
17659            answered: 1,
17660            expected: 1,
17661            clean: true,
17662            progressed: false,
17663            vote_split: false,
17664            reconsideration: Vec::new(),
17665            verdict: None,
17666        };
17667        let state = state_with_round(round);
17668        let body = pr_message(&state, 'A').body;
17669        assert!(!body.contains("Open review findings"), "{body}");
17670        assert!(!body.contains("Declined"), "{body}");
17671    }
17672
17673    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17674        let mut state = RunState::new(
17675            PathBuf::from("/repo"),
17676            "main".to_owned(),
17677            "abc1234".to_owned(),
17678            instruction.to_owned(),
17679            Config::default(),
17680        );
17681        state.candidates.push(Candidate {
17682            index: 0,
17683            label: 'A',
17684            agent: "alpha".to_owned(),
17685            branch: "magi/x/A".to_owned(),
17686            worktree: PathBuf::from("/wt"),
17687            summary: summary.to_owned(),
17688            stat: String::new(),
17689            files: 1,
17690            commits: 1,
17691            empty: false,
17692            failed: None,
17693            verified_noop: None,
17694            folded: false,
17695            duration_ms: 0,
17696        });
17697        state
17698    }
17699
17700    fn review_state(subjects: &[&str]) -> RunState {
17701        let mut state = state_with_summary(
17702            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17703            "",
17704        );
17705        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17706        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17707        state
17708    }
17709
17710    /// A branch rebased onto a main that moved past the recorded
17711    /// `base_commit` is titled from its own first commit, never main's.
17712    #[tokio::test]
17713    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17714        ask_test_home();
17715        let tmp = tempfile::tempdir().unwrap();
17716        let repo = tmp.path().join("repo");
17717        std::fs::create_dir_all(&repo).unwrap();
17718        init_repo(&repo);
17719        let origin = tmp.path().join("origin.git");
17720        let g = |dir: &Path, args: &[&str]| {
17721            let out = std::process::Command::new("git")
17722                .args(args)
17723                .current_dir(dir)
17724                .quiet()
17725                .output()
17726                .expect("spawn git");
17727            assert!(
17728                out.status.success(),
17729                "git {args:?}: {}",
17730                String::from_utf8_lossy(&out.stderr)
17731            );
17732        };
17733        g(
17734            tmp.path(),
17735            &[
17736                "clone",
17737                "--bare",
17738                "-q",
17739                repo.to_str().unwrap(),
17740                origin.to_str().unwrap(),
17741            ],
17742        );
17743        g(
17744            &repo,
17745            &["remote", "add", "origin", origin.to_str().unwrap()],
17746        );
17747        let c1 = git::rev_parse(&repo, "main").await.unwrap();
17748
17749        // Main moves on; the branch is built on top of the new main.
17750        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17751        g(&repo, &["add", "-A"]);
17752        g(
17753            &repo,
17754            &["commit", "-q", "-m", "chore(deps): update a crate"],
17755        );
17756        g(&repo, &["push", "-q", "origin", "main"]);
17757        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17758        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17759        g(&repo, &["add", "-A"]);
17760        g(
17761            &repo,
17762            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17763        );
17764        g(&repo, &["checkout", "-q", "main"]);
17765
17766        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17767        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17768        // Without a readable tracking ref the recorded base's merge base is used.
17769        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17770        assert_eq!(fallback, c1);
17771
17772        let mut state = review_state(&[
17773            "chore(deps): update a crate",
17774            "fix(daemon): apply a chosen action",
17775        ]);
17776        state.repo = repo.clone();
17777        state.base_branch = "main".to_owned();
17778        state.base_commit = c1;
17779        refresh_reviewed_commits(&mut state, "feat/own").await;
17780        assert_eq!(
17781            state.reviewed_commits,
17782            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17783        );
17784        assert_eq!(
17785            review_title(&state).as_deref(),
17786            Some("fix(daemon): apply a chosen action")
17787        );
17788        assert_eq!(
17789            leaked_subjects(&state, "feat/own").await,
17790            Some(vec!["chore(deps): update a crate".to_owned()])
17791        );
17792        // A stale tracking ref is still used when the fetch fails, but the
17793        // leak list is withheld.
17794        state.config.merge.remote = "nowhere".to_owned();
17795        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17796    }
17797
17798    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
17799    /// with `origin/main` one commit behind it, and a summarizing agent that
17800    /// prints whatever `reply` holds.
17801    struct SummaryFixture {
17802        _tmp: tempfile::TempDir,
17803        repo: PathBuf,
17804        reply: PathBuf,
17805        runner: Runner,
17806    }
17807
17808    fn summary_fixture() -> SummaryFixture {
17809        ask_test_home();
17810        let tmp = tempfile::tempdir().unwrap();
17811        let repo = tmp.path().join("repo");
17812        std::fs::create_dir_all(&repo).unwrap();
17813        init_repo(&repo);
17814        let origin = tmp.path().join("origin.git");
17815        let g = |args: &[&str]| {
17816            let out = std::process::Command::new("git")
17817                .args(args)
17818                .current_dir(&repo)
17819                .quiet()
17820                .output()
17821                .expect("spawn git");
17822            assert!(
17823                out.status.success(),
17824                "git {args:?}: {}",
17825                String::from_utf8_lossy(&out.stderr)
17826            );
17827        };
17828        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17829        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17830        g(&["push", "-q", "origin", "main"]);
17831        g(&["checkout", "-q", "-b", "feat/own"]);
17832        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17833        g(&["add", "-A"]);
17834        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17835        let reply = tmp.path().join("reply.txt");
17836        std::fs::write(
17837            &reply,
17838            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17839        )
17840        .unwrap();
17841        let mut runner = runner_at(RunStatus::Gating);
17842        let mut state = review_state(&["feat(own): add the own file"]);
17843        state.repo = repo.clone();
17844        state.base_branch = "main".to_owned();
17845        state.candidates[0].branch = "feat/own".to_owned();
17846        state.candidates[0].worktree = repo.clone();
17847        let mut author = spec("alpha");
17848        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17849        author
17850            .env
17851            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17852        state.config.agents = vec![author];
17853        runner.state = state;
17854        SummaryFixture {
17855            _tmp: tmp,
17856            repo,
17857            reply,
17858            runner,
17859        }
17860    }
17861
17862    async fn posted(f: &mut SummaryFixture) -> PrMessage {
17863        let winner = f.runner.state.candidates[0].clone();
17864        f.runner
17865            .guarded_pr_message(&winner, None, true)
17866            .await
17867            .unwrap()
17868            .expect("an English description never parks")
17869    }
17870
17871    fn clean_round_with_minor_findings() -> ReviewRound {
17872        let mut round = passing_round(2, None);
17873        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17874        round.reviews = vec![ReviewRecord {
17875            attempts: 0,
17876            reviewer: 1,
17877            agent: "alpha".into(),
17878            summary: "内部レビュー".into(),
17879            findings: vec![
17880                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17881                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17882            ],
17883            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17884            failed: None,
17885            duration_ms: 0,
17886        }];
17887        round
17888    }
17889
17890    #[tokio::test]
17891    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17892        for review_only in [false, true] {
17893            let mut f = summary_fixture();
17894            if !review_only {
17895                f.runner.state.candidates[0].agent = "alpha".into();
17896            }
17897            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17898            assert!(f.runner.state.reviews[0].clean);
17899            assert!(f.runner.state.reviews[0].fix.is_none());
17900            assert!(f.runner.state.open_findings().is_empty());
17901            assert_eq!(
17902                crate::pr_outcome::required_finding_ids(&f.runner.state),
17903                ["R2-1-1", "R2-1-2"]
17904            );
17905            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17906            let evidence = crate::pr_outcome::summary_evidence(
17907                &f.runner.state,
17908                crate::pr_outcome::FinalHead::At(&head),
17909            );
17910            assert!(
17911                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17912            );
17913            // The ordinary reply omits both IDs: no cached description is adopted.
17914            let omitted = posted(&mut f).await;
17915            assert!(omitted.body.contains("No description"));
17916            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17917            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17918            assert!(f.runner.state.change_summary.is_none());
17919            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17920            let described = posted(&mut f).await;
17921            assert!(described.body.contains("improve documentation for callers"));
17922            assert!(described.body.contains("remaining edge case"));
17923            assert!(described.body.contains("## Open review findings"));
17924            assert!(!described.body.contains("未修正"));
17925        }
17926    }
17927
17928    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17929        let mut f = summary_fixture();
17930        if !review_only {
17931            f.runner.state.candidates[0].agent = "alpha".into();
17932        }
17933        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17934        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17935        fixed
17936            .fix
17937            .as_mut()
17938            .unwrap()
17939            .rejected
17940            .push(crate::verdict::Rejection {
17941                id: "R1-1-1".into(),
17942                why: "呼び出されないコードのため辞退".into(),
17943            });
17944        let mut clean = clean_round_with_minor_findings();
17945        clean.verified_head = Some(head.clone());
17946        f.runner.state.reviews = vec![fixed, clean];
17947        ok_gate(&mut f.runner.state, Some(&head));
17948        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17949        f.runner.state.config.agents.push(scripted("beta", reply));
17950        f.runner.state.config.roles.language_judge =
17951            Some(crate::config::AgentChoice::One("beta".into()));
17952        f
17953    }
17954
17955    #[tokio::test]
17956    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17957        for review_only in [false, true] {
17958            let mut f = rewrite_fixture_with_findings(review_only,
17959                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17960            let m = posted(&mut f).await;
17961            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17962            let expected =
17963                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17964            assert!(m.body.contains(&expected));
17965            assert!(m.body.contains("## Review rounds"));
17966            assert!(
17967                m.body
17968                    .contains("Gate (`verify.gate`): passed on the final commit")
17969            );
17970            assert!(
17971                m.body
17972                    .contains("Tests (`verify.e2e`): passed on the final commit")
17973            );
17974            let c = f.runner.state.change_summary.as_ref().unwrap();
17975            assert!(
17976                m.body
17977                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17978            );
17979            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17980            assert!(crate::pr_outcome::covers_required_findings(
17981                &f.runner.state,
17982                &m.body
17983            ));
17984            assert!(m.body.contains("documentation remains incomplete"));
17985            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17986            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17987            assert!(f.runner.state.github_text.is_none());
17988        }
17989    }
17990
17991    #[tokio::test]
17992    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17993        for review_only in [false, true] {
17994            for reply in [
17995                "TITLE: fix: public description\nA short English description.",
17996                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17997            ] {
17998                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17999                let winner = f.runner.state.candidates[0].clone();
18000                let posted = f
18001                    .runner
18002                    .guarded_pr_message(&winner, None, true)
18003                    .await
18004                    .unwrap();
18005                assert!(posted.is_none());
18006                assert!(f.runner.state.parked);
18007                let withheld = f.runner.state.github_text.as_ref().unwrap();
18008                assert!(
18009                    withheld.rewrite.is_none(),
18010                    "an incomplete rewrite must not be selectable"
18011                );
18012                let q = ask::Questions::open()
18013                    .get(withheld.question.as_ref().unwrap())
18014                    .unwrap();
18015                assert!(
18016                    !q.choices
18017                        .contains(&crate::github_text::USE_REWRITE.to_owned())
18018                );
18019                assert!(
18020                    q.detail.contains("R1-1-1")
18021                        && q.detail.contains("R2-1-1")
18022                        && q.detail.contains("R2-1-2")
18023                );
18024            }
18025        }
18026    }
18027
18028    #[tokio::test]
18029    async fn japanese_reports_stay_local_in_both_public_run_modes() {
18030        for review_only in [false, true] {
18031            let mut f = summary_fixture();
18032            if !review_only {
18033                f.runner.state.candidates[0].agent = "alpha".into();
18034            }
18035            f.runner.state.instruction = "内部タスクの説明".into();
18036            f.runner.state.candidates[0].summary = "内部の候補報告".into();
18037            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18038            f.runner.state.reviews = vec![
18039                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18040                round_with_notes(2, Some("二回目の内部修正報告です。")),
18041                passing_round(3, Some(&head)),
18042            ];
18043            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18044            ok_gate(&mut f.runner.state, Some(&head));
18045            let captured = f.repo.join("public-prompt.txt");
18046            let author = &mut f.runner.state.config.agents[0];
18047            author.command = vec![
18048                "sh".into(),
18049                "-c".into(),
18050                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18051            ];
18052            author.env.insert(
18053                "PROMPT_FILE".into(),
18054                captured.to_string_lossy().into_owned(),
18055            );
18056            let m = posted(&mut f).await;
18057            assert!(m.body.contains("Adds `own.txt`"));
18058            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18059            assert!(m.body.contains("R1-1-1"));
18060            assert!(
18061                m.body
18062                    .contains("Gate (`verify.gate`): passed on the final commit")
18063            );
18064            assert!(
18065                m.body
18066                    .contains("Tests (`verify.e2e`): passed on the final commit")
18067            );
18068            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18069            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18070            let prompt = std::fs::read_to_string(captured).unwrap();
18071            assert!(prompt.contains(&format!(
18072                "git diff {} {head}",
18073                f.runner.state.change_summary.as_ref().unwrap().base
18074            )));
18075            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18076            assert_eq!(
18077                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18078                "二回目の内部修正報告です。"
18079            );
18080        }
18081    }
18082
18083    #[tokio::test]
18084    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18085        for review_only in [false, true] {
18086            let mut f = summary_fixture();
18087            if !review_only {
18088                f.runner.state.candidates[0].agent = "alpha".into();
18089            }
18090            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18091            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18092            std::fs::write(&f.reply, "no usable description").unwrap();
18093            let failed = posted(&mut f).await;
18094            assert!(
18095                failed
18096                    .body
18097                    .contains("No description of what this change does could be generated")
18098            );
18099            assert!(failed.body.is_ascii());
18100            assert!(f.runner.state.github_text.is_none());
18101            assert!(f.runner.state.change_summary.is_none());
18102            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18103            let evidence = crate::pr_outcome::summary_evidence(
18104                &f.runner.state,
18105                crate::pr_outcome::FinalHead::At(&head),
18106            );
18107            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18108            assert!(
18109                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18110                    .exists()
18111            );
18112            f.runner.state =
18113                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18114            std::fs::write(
18115                &f.reply,
18116                "TITLE: feat(own): add the own file\nAdds the committed own file.",
18117            )
18118            .unwrap();
18119            let retried = posted(&mut f).await;
18120            assert!(retried.body.contains("Adds the committed own file."));
18121            assert_eq!(
18122                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18123                "日本語の修正報告"
18124            );
18125        }
18126    }
18127
18128    #[tokio::test]
18129    async fn same_head_verification_and_findings_invalidate_public_prose() {
18130        let mut f = summary_fixture();
18131        posted(&mut f).await;
18132        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18133        let original = f
18134            .runner
18135            .state
18136            .change_summary
18137            .as_ref()
18138            .unwrap()
18139            .evidence
18140            .clone();
18141        ok_gate(&mut f.runner.state, Some(&head));
18142        std::fs::write(
18143            &f.reply,
18144            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18145        )
18146        .unwrap();
18147        let verified = posted(&mut f).await;
18148        assert!(verified.body.contains("now verified"));
18149        assert_ne!(
18150            f.runner.state.change_summary.as_ref().unwrap().evidence,
18151            original
18152        );
18153        let mut round = round_with_notes(1, Some("内部報告"));
18154        round.reviews.push(ReviewRecord {
18155            attempts: 0,
18156            reviewer: 1,
18157            agent: "alpha".into(),
18158            summary: "内部レビュー".into(),
18159            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18160            vote: None,
18161            failed: None,
18162            duration_ms: 0,
18163        });
18164        round
18165            .fix
18166            .as_mut()
18167            .unwrap()
18168            .rejected
18169            .push(crate::verdict::Rejection {
18170                id: "R1-1-1".into(),
18171                why: "呼び出し元から到達できない".into(),
18172            });
18173        f.runner.state.reviews = vec![round];
18174        // A description omitting the required ID is unusable, even with a title.
18175        let omitted = posted(&mut f).await;
18176        assert!(omitted.body.contains("No description"));
18177        assert!(omitted.body.contains("R1-1-1"));
18178        assert!(!omitted.body.contains("到達"));
18179        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18180        let described = posted(&mut f).await;
18181        assert!(described.body.contains("unreachable from callers"));
18182        assert!(described.body.contains("R1-1-1"));
18183        assert!(!described.body.contains("到達"));
18184        let evidence = f
18185            .runner
18186            .state
18187            .change_summary
18188            .as_ref()
18189            .unwrap()
18190            .evidence
18191            .clone();
18192        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18193        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18194        assert!(posted(&mut f).await.body.contains("rationale updated"));
18195        assert_ne!(
18196            f.runner.state.change_summary.as_ref().unwrap().evidence,
18197            evidence
18198        );
18199        // Old serialized summaries have no evidence fingerprint and must regenerate.
18200        f.runner
18201            .state
18202            .change_summary
18203            .as_mut()
18204            .unwrap()
18205            .evidence
18206            .clear();
18207        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18208        assert!(posted(&mut f).await.body.contains("current specification"));
18209    }
18210
18211    #[tokio::test]
18212    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18213        let mut f = summary_fixture();
18214        let m = posted(&mut f).await;
18215        assert_eq!(m.title, "feat(own): add the own file");
18216        assert!(
18217            m.body
18218                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
18219            "{}",
18220            m.body
18221        );
18222        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
18223        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
18224        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18225        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18226        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
18227        let kept = f.runner.state.change_summary.as_ref().unwrap();
18228        assert_eq!(
18229            (kept.head.as_str(), kept.base.as_str()),
18230            (head.as_str(), base.as_str())
18231        );
18232        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18233    }
18234
18235    #[tokio::test]
18236    async fn review_only_summary_receives_the_configured_github_language() {
18237        let mut f = summary_fixture();
18238        f.runner.state.config.graph.github_language = "ja".into();
18239        let captured = f.repo.join("summary-prompt.txt");
18240        let author = &mut f.runner.state.config.agents[0];
18241        author.command = vec![
18242            "sh".into(),
18243            "-c".into(),
18244            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18245        ];
18246        author.env.insert(
18247            "PROMPT_FILE".into(),
18248            captured.to_string_lossy().into_owned(),
18249        );
18250        std::fs::write(
18251            &f.reply,
18252            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18253        )
18254        .unwrap();
18255        let m = posted(&mut f).await;
18256        let prompt = std::fs::read_to_string(captured).unwrap();
18257        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18258        assert!(!prompt.contains("in English"));
18259        assert_eq!(m.title, "feat(own): ファイルを追加");
18260        assert!(m.body.contains("own.txt を追加します。"));
18261    }
18262
18263    #[tokio::test]
18264    async fn review_fixes_come_after_the_whole_change_summary() {
18265        let mut f = summary_fixture();
18266        f.runner.state.reviews = vec![
18267            round_with_notes(1, Some("handled the PATH case")),
18268            round_with_notes(2, None),
18269        ];
18270        let m = posted(&mut f).await;
18271        let summary = m.body.find("## Summary").expect("summary");
18272        let fixes = m.body.find("## Review rounds").expect("review rounds");
18273        assert!(summary < fixes, "{}", m.body);
18274        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18275        assert!(m.body.contains("## Final verification"), "{}", m.body);
18276    }
18277
18278    #[tokio::test]
18279    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18280        let mut f = summary_fixture();
18281        let first = posted(&mut f).await;
18282        let first_head = f.runner.state.change_summary.clone().unwrap().head;
18283        // The same head reuses the record: a different reply is not read.
18284        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18285        let again = posted(&mut f).await;
18286        assert_eq!((again.title, again.body), (first.title, first.body));
18287
18288        let g = |args: &[&str]| {
18289            let out = std::process::Command::new("git")
18290                .args(args)
18291                .current_dir(&f.repo)
18292                .quiet()
18293                .output()
18294                .expect("spawn git");
18295            assert!(out.status.success(), "git {args:?}");
18296        };
18297        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18298        g(&["add", "-A"]);
18299        g(&["commit", "-q", "-m", "fix(own): handle more"]);
18300        let second = posted(&mut f).await;
18301        let kept = f.runner.state.change_summary.clone().unwrap();
18302        assert_ne!(kept.head, first_head);
18303        assert!(second.body.contains("Not read yet."), "{}", second.body);
18304        assert!(!second.body.contains("so the first summary is read"));
18305        assert_eq!(second.title, "feat(own): other");
18306    }
18307
18308    #[tokio::test]
18309    async fn an_unavailable_summary_is_stated_and_never_invented() {
18310        let mut f = summary_fixture();
18311        std::fs::write(&f.reply, "no title line here").unwrap();
18312        let m = posted(&mut f).await;
18313        assert!(
18314            m.body
18315                .contains("No description of what this change does could be generated"),
18316            "{}",
18317            m.body
18318        );
18319        assert!(m.body.contains("internal reports remain in local records"));
18320        assert!(f.runner.state.change_summary.is_none());
18321        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18322        // A preview asks nobody and claims nothing.
18323        let winner = f.runner.state.candidates[0].clone();
18324        let preview = f
18325            .runner
18326            .guarded_pr_message(&winner, None, false)
18327            .await
18328            .unwrap()
18329            .unwrap();
18330        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18331    }
18332
18333    #[test]
18334    fn pr_message_review_single_commit_uses_its_subject() {
18335        let state = review_state(&["feat(nats): per-role user"]);
18336        let m = pr_message(&state, 'A');
18337        assert_eq!(m.title, "feat(nats): per-role user");
18338        assert!(!m.body.contains("Review the work already"), "{}", m.body);
18339        assert!(m.body.contains("## Commits under review"), "{}", m.body);
18340    }
18341
18342    #[test]
18343    fn pr_message_review_multi_commit_takes_the_oldest() {
18344        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18345        let m = pr_message(&state, 'A');
18346        assert_eq!(m.title, "feat: the change");
18347        for s in ["feat: the change", "fix: typo", "fix: again"] {
18348            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18349        }
18350    }
18351
18352    #[test]
18353    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18354        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18355            let mut state = review_state(&[first, "fix: later fixup"]);
18356            state.candidates[0].branch = "機能/ログイン".to_owned();
18357            let m = pr_message(&state, 'A');
18358            assert!(
18359                m.title.starts_with("chore: land candidate A of run"),
18360                "{}",
18361                m.title
18362            );
18363        }
18364    }
18365
18366    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18367        BranchFacts {
18368            commits: commits
18369                .iter()
18370                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18371                .collect(),
18372            stat: stat.to_owned(),
18373        }
18374    }
18375
18376    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18377        let mut r = review_round(true, 0, 1, 1, true, true);
18378        r.round = round;
18379        r.fix = notes.map(|n| FixRecord {
18380            agent: "fixer".to_owned(),
18381            addressed: Vec::new(),
18382            rejected: Vec::new(),
18383            notes: n.to_owned(),
18384            committed: true,
18385            failed: None,
18386            duration_ms: 0,
18387            continuation: None,
18388        });
18389        r
18390    }
18391
18392    #[test]
18393    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18394        let state = review_state(&["ignored"]);
18395        let f = facts(
18396            &[
18397                (
18398                    "fix(login): resolve PATH on macOS",
18399                    "Login shells skip rc files.",
18400                ),
18401                ("fix: address review", ""),
18402            ],
18403            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18404        );
18405        let m = pr_message_with(&state, 'A', Some(&f));
18406        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18407        assert!(
18408            m.body
18409                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
18410            "{}",
18411            m.body
18412        );
18413        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18414        assert!(m.body.contains("## Diff stat"), "{}", m.body);
18415        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18416        for banned in [
18417            "Review the work already",
18418            "no task statement",
18419            "Original task",
18420        ] {
18421            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18422        }
18423        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18424    }
18425
18426    #[test]
18427    fn pr_message_review_truncates_a_huge_first_commit_body() {
18428        let state = review_state(&["ignored"]);
18429        let f = facts(
18430            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18431            "s",
18432        );
18433        let m = pr_message_with(&state, 'A', Some(&f));
18434        assert!(m.body.len() < 30_000, "{}", m.body.len());
18435        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18436        assert!(
18437            m.body.contains("- ... 1 more commit(s)"),
18438            "{}",
18439            m.body.len()
18440        );
18441        assert!(m.body.ends_with("magi:candidate-a\n"));
18442    }
18443
18444    #[test]
18445    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18446        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18447        assert_eq!(m.title, "feat: x");
18448        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18449        assert!(!m.body.contains("Diff stat"), "{}", m.body);
18450        assert!(!m.body.contains("no task statement"), "{}", m.body);
18451    }
18452
18453    #[test]
18454    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18455        let mut state = review_state(&["日本語の件名"]);
18456        state.candidates[0].branch = "fix/macos-login-path".to_owned();
18457        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18458        state.candidates[0].branch = "機能/ログイン".to_owned();
18459        assert!(
18460            pr_message(&state, 'A')
18461                .title
18462                .starts_with("chore: land candidate A")
18463        );
18464    }
18465
18466    #[test]
18467    fn pr_message_review_fixes_survive_a_clean_final_round() {
18468        let mut state = review_state(&["feat: x"]);
18469        state.reviews = vec![
18470            round_with_notes(1, Some("handled the PATH case")),
18471            round_with_notes(2, None),
18472        ];
18473        let body = pr_message(&state, 'A').body;
18474        assert!(
18475            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18476            "{body}"
18477        );
18478        assert!(body.contains("### Round 1"), "{body}");
18479
18480        state.reviews = vec![
18481            round_with_notes(1, Some("first fix")),
18482            round_with_notes(2, Some("")),
18483            round_with_notes(3, Some("second fix")),
18484            round_with_notes(4, None),
18485        ];
18486        let body = pr_message(&state, 'A').body;
18487        assert!(
18488            body.contains("### Round 1") && !body.contains("first fix"),
18489            "{body}"
18490        );
18491        assert!(
18492            body.contains("### Round 3") && !body.contains("second fix"),
18493            "{body}"
18494        );
18495        assert!(body.contains("### Round 2"), "{body}");
18496    }
18497
18498    const FINAL: &str = "abcdef1234567890";
18499
18500    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18501        let mut r = round_with_notes(round, None);
18502        r.verified_head = verified.map(str::to_owned);
18503        r
18504    }
18505
18506    fn ok_gate(state: &mut RunState, head: Option<&str>) {
18507        state.gate = vec![CommandOutcome {
18508            command: "gate".to_owned(),
18509            code: Some(0),
18510            output_tail: String::new(),
18511            duration_ms: 0,
18512            resource_blocked: false,
18513        }];
18514        state.gate_ran = true;
18515        state.gate_head = head.map(str::to_owned);
18516    }
18517
18518    fn body_at(state: &RunState) -> String {
18519        pr_message_raw(
18520            state,
18521            'A',
18522            None,
18523            crate::pr_outcome::FinalHead::At(FINAL),
18524            ChangeNote::NotAsked,
18525        )
18526        .body
18527    }
18528
18529    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18530
18531    #[test]
18532    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18533        let mut state = review_state(&["feat: x"]);
18534        state.reviews = vec![
18535            round_with_notes(1, Some(SANDBOXED)),
18536            passing_round(2, Some(FINAL)),
18537        ];
18538        ok_gate(&mut state, Some(FINAL));
18539        let body = body_at(&state);
18540        assert!(!body.contains("Handled the PATH case."), "{body}");
18541        assert!(!body.contains("could not commit"), "{body}");
18542        assert!(!body.contains("could not run"), "{body}");
18543        assert!(
18544            body.contains("The fix is committed on the branch"),
18545            "{body}"
18546        );
18547        assert!(!body.contains("sandbox"), "{body}");
18548        assert!(
18549            body.contains("Tests (`verify.e2e`): passed on the final commit"),
18550            "{body}"
18551        );
18552        assert!(
18553            body.contains("Gate (`verify.gate`): passed on the final commit"),
18554            "{body}"
18555        );
18556        // The raw report is untouched in the record.
18557        assert!(
18558            state.reviews[0]
18559                .fix
18560                .as_ref()
18561                .unwrap()
18562                .notes
18563                .contains("could not commit")
18564        );
18565    }
18566
18567    #[test]
18568    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18569        let mut state = review_state(&["feat: x"]);
18570        let mut r1 = round_with_notes(1, Some(SANDBOXED));
18571        r1.fix.as_mut().unwrap().committed = false;
18572        let mut r2 = passing_round(2, Some(FINAL));
18573        r2.e2e[0].code = Some(1);
18574        state.reviews = vec![r1, r2];
18575        let body = body_at(&state);
18576        assert!(!body.contains("could not commit"), "{body}");
18577        assert!(!body.contains("could not run"), "{body}");
18578        assert!(body.contains("This round produced no commit"), "{body}");
18579        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18580        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18581        assert!(!body.contains("passed"), "{body}");
18582    }
18583
18584    #[test]
18585    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18586        let mut state = review_state(&["feat: x"]);
18587        state.reviews = vec![
18588            round_with_notes(1, Some(SANDBOXED)),
18589            passing_round(2, Some("0123456789abcdef")),
18590        ];
18591        ok_gate(&mut state, Some("0123456789abcdef"));
18592        let body = body_at(&state);
18593        assert!(!body.contains("HTTP tests could not run"), "{body}");
18594        // The commit claim is still settled by the structured record.
18595        assert!(!body.contains("could not commit"), "{body}");
18596        assert!(
18597            body.contains("passed on an earlier commit (`0123456`)"),
18598            "{body}"
18599        );
18600        assert!(!body.contains("passed on the final commit"), "{body}");
18601
18602        // A gate recorded before `gate_head` existed is "not recorded".
18603        ok_gate(&mut state, None);
18604        let body = body_at(&state);
18605        assert!(
18606            body.contains("passed on a commit that was not recorded"),
18607            "{body}"
18608        );
18609    }
18610
18611    #[test]
18612    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18613        let state = state_with_summary(
18614            "add retries to the client",
18615            "TITLE: feat: retries\n\nDid it.",
18616        );
18617        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18618        assert_eq!(m.title, "feat: retries");
18619        assert!(
18620            m.body.contains("<summary>Original task</summary>"),
18621            "{}",
18622            m.body
18623        );
18624        assert!(!m.body.contains("Commits under review"), "{}", m.body);
18625        assert!(
18626            m.body
18627                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18628            "{}",
18629            m.body
18630        );
18631    }
18632
18633    #[test]
18634    fn pr_message_review_bounds_a_long_english_subject() {
18635        let long = format!("feat: {}", "word ".repeat(100));
18636        let m = pr_message(&review_state(&[&long]), 'A');
18637        assert!(m.title.starts_with("feat: word"), "{}", m.title);
18638        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18639    }
18640
18641    #[test]
18642    fn pr_message_implementation_run_is_unchanged_by_review_support() {
18643        let state = state_with_summary("add retries\n\ndetails", "- did some things");
18644        let m = pr_message(&state, 'A');
18645        assert_eq!(m.title, "add retries");
18646        assert!(m.body.contains("<summary>Original task</summary>"));
18647        assert!(!m.body.contains("Commits under review"));
18648        assert_eq!(landing_subject_source(&state), state.instruction);
18649    }
18650
18651    #[test]
18652    fn review_run_squash_subject_is_the_change_not_the_prompt() {
18653        let state = review_state(&["feat: the change", "fix: typo"]);
18654        let source = landing_subject_source(&state);
18655        assert_eq!(land::merge_subject("", &source), "feat: the change");
18656        assert_eq!(
18657            land::merge_subject("magi: candidate A (uncommitted work)", &source),
18658            "feat: the change"
18659        );
18660        // An operator's rename still wins.
18661        assert_eq!(
18662            land::merge_subject("feat: renamed by hand", &source),
18663            "feat: renamed by hand"
18664        );
18665        let blank = review_state(&["日本語"]);
18666        assert!(
18667            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18668        );
18669    }
18670
18671    #[test]
18672    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18673        let state = review_state(&["feat: the change"]);
18674        let source = landing_subject_source(&state);
18675        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18676        assert_eq!(
18677            land::merge_subject(landing_title(&state, old), &source),
18678            "feat: the change"
18679        );
18680        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18681        let task = state_with_summary("add retries", "");
18682        assert_eq!(landing_title(&task, old), old);
18683    }
18684
18685    #[test]
18686    fn pr_message_describes_the_change_not_the_task() {
18687        let state = state_with_summary(
18688            "今回やってほしいこと: results projector を直す",
18689            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18690        );
18691        let m = pr_message(&state, 'A');
18692        assert_eq!(m.title, "fix(web): batch the runs list reads");
18693        assert!(
18694            m.body.starts_with("## Summary\n\n- reads run.json once"),
18695            "{}",
18696            m.body
18697        );
18698        assert!(!m.body.contains("TITLE:"), "{}", m.body);
18699        let task_at = m.body.find("今回やってほしいこと").unwrap();
18700        let details_at = m.body.find("<details>").unwrap();
18701        assert!(
18702            details_at < task_at,
18703            "the task lives inside <details>: {}",
18704            m.body
18705        );
18706        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18707        assert!(m.body.contains("magi:candidate-a"));
18708    }
18709
18710    #[test]
18711    fn pr_message_falls_back_to_the_task_without_a_title_line() {
18712        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18713        let m = pr_message(&state, 'A');
18714        assert_eq!(m.title, "add retries");
18715        assert!(
18716            m.body.contains("## Summary\n\n- did some things"),
18717            "{}",
18718            m.body
18719        );
18720
18721        let none = RunState::new(
18722            PathBuf::from("/repo"),
18723            "main".to_owned(),
18724            "abc1234".to_owned(),
18725            "add retries".to_owned(),
18726            Config::default(),
18727        );
18728        let m = pr_message(&none, 'A');
18729        assert_eq!(m.title, "add retries");
18730        assert!(!m.body.contains("## Summary"), "{}", m.body);
18731    }
18732
18733    #[test]
18734    fn pr_message_refuses_the_candidate_commit_subject() {
18735        for bad in [
18736            "TITLE: magi: candidate A (uncommitted work)",
18737            "TITLE: chore: stuff (uncommitted work)",
18738            "TITLE:   ",
18739        ] {
18740            let state = state_with_summary("add retries", bad);
18741            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18742        }
18743    }
18744
18745    #[test]
18746    fn pr_message_bounds_a_very_long_task_and_title() {
18747        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18748        let state = state_with_summary(&long, "- nothing");
18749        let m = pr_message(&state, 'A');
18750        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18751        assert!(!m.title.contains('\n'));
18752
18753        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18754        let m = pr_message(&state, 'A');
18755        assert!(m.title.starts_with("feat: "));
18756        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18757        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18758    }
18759
18760    fn long_title_of(instruction: &str) -> String {
18761        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18762    }
18763
18764    #[test]
18765    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18766        let first = "Make the landing path keep a readable title for long tasks";
18767        let line = format!(
18768            "{first}. {}",
18769            "And then keep going with more words ".repeat(20)
18770        );
18771        let t = long_title_of(&line);
18772        assert_eq!(t, first);
18773        assert!(!t.starts_with("chore: land"));
18774    }
18775
18776    #[test]
18777    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18778        let line = "word ".repeat(200);
18779        let t = long_title_of(&line);
18780        assert!(t.ends_with("word..."), "{t}");
18781        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18782    }
18783
18784    #[test]
18785    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18786        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18787            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18788        }
18789    }
18790
18791    #[test]
18792    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18793        let mut state = state_with_summary("再試行を追加する", "- nothing");
18794        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18795        state.config.graph.github_language = "ja".into();
18796        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18797    }
18798
18799    #[test]
18800    fn pr_message_title_limit_is_exact() {
18801        let at = "a".repeat(PR_TITLE_MAX);
18802        assert_eq!(long_title_of(&at), at);
18803        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18804        assert!(over.ends_with("..."), "{over}");
18805        assert_eq!(over.chars().count(), PR_TITLE_MAX);
18806    }
18807
18808    #[test]
18809    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18810        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18811        let t = long_title_of(&line);
18812        assert!(t.ends_with("..."), "{t}");
18813        assert!(t.is_ascii(), "{t}");
18814    }
18815
18816    #[test]
18817    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18818        let line = format!(
18819            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18820            "plus more filler words ".repeat(20)
18821        );
18822        let t = long_title_of(&line);
18823        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18824    }
18825
18826    #[test]
18827    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18828        let line = format!("\n\n# {}", "title words ".repeat(40));
18829        let state = state_with_summary(&line, "- nothing");
18830        let m = pr_message(&state, 'A');
18831        assert!(m.title.starts_with("title words"), "{}", m.title);
18832        assert_eq!(
18833            land::merge_subject(&m.title, &landing_subject_source(&state)),
18834            m.title
18835        );
18836        // An operator's rename wins untouched.
18837        assert_eq!(
18838            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18839            "feat: renamed by hand"
18840        );
18841    }
18842
18843    #[tokio::test]
18844    async fn github_text_rewrite_is_bounded_and_falls_back() {
18845        crate::run::pin_test_home();
18846        for (reply, accepted) in [
18847            (
18848                "TITLE: fix: retries\nAdd retries for failed requests.",
18849                true,
18850            ),
18851            (
18852                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18853                false,
18854            ),
18855            (
18856                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18857                true,
18858            ),
18859            (
18860                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18861                true,
18862            ),
18863            (
18864                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18865                true,
18866            ),
18867            (
18868                "TITLE: fix: retries\n日本語の説明です。 token=secret",
18869                false,
18870            ),
18871            ("TITLE: fix: retries\n", false),
18872        ] {
18873            let dir = tempfile::tempdir().unwrap();
18874            let mut runner = runner_at(RunStatus::Gating);
18875            runner.state = state_with_summary(
18876                "add retries",
18877                "TITLE: fix: retries\n日本語の説明を書きます。",
18878            );
18879            runner.state.repo = dir.path().to_owned();
18880            runner.state.candidates[0].worktree = dir.path().to_owned();
18881            let mut author = spec("alpha");
18882            author.command = vec![
18883                "sh".into(),
18884                "-c".into(),
18885                "printf '%s' \"$REWRITE_REPLY\"".into(),
18886            ];
18887            author.env.insert("REWRITE_REPLY".into(), reply.into());
18888            runner.state.config.agents = vec![author];
18889            seed_public_summary(&mut runner);
18890            let winner = runner.state.candidates[0].clone();
18891            let first = runner
18892                .guarded_pr_message(&winner, None, true)
18893                .await
18894                .unwrap();
18895            let message = if accepted {
18896                first.expect("a passing rewrite never asks")
18897            } else {
18898                // Withheld: the run parks on a question and posts nothing yet.
18899                assert!(first.is_none());
18900                assert!(runner.state.parked);
18901                let store = ask::Questions::open();
18902                let asked: Vec<_> = store
18903                    .list()
18904                    .into_iter()
18905                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18906                    .collect();
18907                assert_eq!(asked.len(), 1);
18908                assert!(!asked[0].detail.contains("secret"));
18909                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18910                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18911                assert_eq!(
18912                    asked[0]
18913                        .choices
18914                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18915                    rewrite.is_some()
18916                );
18917                // Silence: the question lapses and the neutral text is used,
18918                // without a second rewrite call or a second question.
18919                store
18920                    .update(&asked[0].id, |q| {
18921                        q.abandon("test");
18922                        Ok(())
18923                    })
18924                    .unwrap();
18925                let again = runner
18926                    .guarded_pr_message(&winner, None, true)
18927                    .await
18928                    .unwrap()
18929                    .expect("an abandoned question falls back");
18930                let n = store
18931                    .list()
18932                    .into_iter()
18933                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18934                    .count();
18935                assert_eq!(n, 1, "asked once per rejected text");
18936                again
18937            };
18938            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18939            assert!(!message.body.contains("secret"));
18940            assert!(!message.body.contains("prefix"));
18941            assert!(!message.title.contains("secret"));
18942            assert_eq!(
18943                message.body.contains("Add retries for failed requests."),
18944                accepted
18945            );
18946            assert_eq!(
18947                runner.state.seats["github-text-rewrite"].turns, 1,
18948                "only one rewrite invocation"
18949            );
18950            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18951            if accepted {
18952                assert!(runner.state.github_text.is_none());
18953                assert!(!runner.state.parked);
18954                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18955                assert!(message.body.contains("## Final verification"));
18956                assert!(message.body.contains("Evidence: final diff"));
18957                assert!(
18958                    message
18959                        .body
18960                        .contains(&format!("magi:run/{}", runner.state.id))
18961                );
18962            }
18963            if !accepted {
18964                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18965                assert!(
18966                    message
18967                        .body
18968                        .contains(&format!("magi:run/{}", runner.state.id))
18969                );
18970            }
18971        }
18972    }
18973
18974    fn scripted(id: &str, reply: &str) -> AgentSpec {
18975        let mut a = spec(id);
18976        a.command = vec![
18977            "sh".into(),
18978            "-c".into(),
18979            "printf '%s' \"$REWRITE_REPLY\"".into(),
18980        ];
18981        a.env.insert("REWRITE_REPLY".into(), reply.into());
18982        a
18983    }
18984
18985    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18986    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18987
18988    /// Gate tests inject generated public output, not internal reports.
18989    fn seed_public_summary(runner: &mut Runner) {
18990        let repo = &runner.state.repo;
18991        let g = |args: &[&str]| {
18992            let out = std::process::Command::new("git")
18993                .args(args)
18994                .current_dir(repo)
18995                .quiet()
18996                .output()
18997                .unwrap();
18998            assert!(out.status.success(), "git {args:?}");
18999            String::from_utf8_lossy(&out.stdout).trim().to_owned()
19000        };
19001        g(&["init", "-q", "-b", "main"]);
19002        g(&[
19003            "-c",
19004            "user.name=Test",
19005            "-c",
19006            "user.email=test@example.invalid",
19007            "commit",
19008            "-q",
19009            "--allow-empty",
19010            "-m",
19011            "feat: fixture",
19012        ]);
19013        let head = g(&["rev-parse", "HEAD"]);
19014        g(&["update-ref", "refs/remotes/origin/main", &head]);
19015        let evidence = crate::pr_outcome::summary_evidence(
19016            &runner.state,
19017            crate::pr_outcome::FinalHead::At(&head),
19018        );
19019        let generated = &runner.state.candidates[0].summary;
19020        runner.state.change_summary = Some(crate::run::ChangeSummary {
19021            base: head.clone(),
19022            head,
19023            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
19024            title: summary_title(generated).unwrap(),
19025            body: summary_without_title(generated),
19026        });
19027    }
19028
19029    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19030        let mut runner = runner_at(RunStatus::Gating);
19031        runner.state = state_with_summary("add retries", JA_SUMMARY);
19032        if review_only {
19033            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19034        }
19035        runner.state.repo = dir.to_owned();
19036        runner.state.candidates[0].worktree = dir.to_owned();
19037        runner.state.config.agents = agents;
19038        seed_public_summary(&mut runner);
19039        runner
19040    }
19041
19042    fn gate_questions(runner: &Runner) -> usize {
19043        ask::Questions::open()
19044            .list()
19045            .into_iter()
19046            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19047            .count()
19048    }
19049
19050    #[tokio::test]
19051    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19052        crate::run::pin_test_home();
19053        let dir = tempfile::tempdir().unwrap();
19054        let mut runner = rewrite_runner(
19055            dir.path(),
19056            true,
19057            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19058        );
19059        runner.state.config.roles.language_judge =
19060            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19061        let winner = runner.state.candidates[0].clone();
19062        let msg = runner
19063            .guarded_pr_message(&winner, None, true)
19064            .await
19065            .unwrap()
19066            .expect("an English rewrite posts without asking");
19067        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19068        assert!(msg.body.contains("<details>kept</details>"));
19069        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19070        assert_eq!(gate_questions(&runner), 0);
19071        assert!(runner.state.github_text.is_none());
19072        assert!(!runner.state.seats.contains_key("impl-A"));
19073        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19074    }
19075
19076    #[tokio::test]
19077    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19078        crate::run::pin_test_home();
19079        let dir = tempfile::tempdir().unwrap();
19080        let mut runner = rewrite_runner(
19081            dir.path(),
19082            false,
19083            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19084        );
19085        runner.state.config.roles.language_judge =
19086            Some(crate::config::AgentChoice::One("beta".into()));
19087        runner.state.seats.insert(
19088            "impl-A".into(),
19089            SeatState::new("impl-A", "alpha", runner.state.seed),
19090        );
19091        let winner = runner.state.candidates[0].clone();
19092        let msg = runner
19093            .guarded_pr_message(&winner, None, true)
19094            .await
19095            .unwrap()
19096            .expect("rewritten");
19097        assert!(msg.body.contains("<details>kept</details>"));
19098        assert_eq!(runner.state.seats["impl-A"].turns, 0);
19099        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19100        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19101    }
19102
19103    #[tokio::test]
19104    async fn rewrite_reuses_a_resumable_implementer_session() {
19105        crate::run::pin_test_home();
19106        let dir = tempfile::tempdir().unwrap();
19107        let mut runner = rewrite_runner(
19108            dir.path(),
19109            false,
19110            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19111        );
19112        runner.state.config.roles.language_judge =
19113            Some(crate::config::AgentChoice::One("beta".into()));
19114        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19115        seat.turns = 1;
19116        runner.state.seats.insert("impl-A".into(), seat);
19117        let winner = runner.state.candidates[0].clone();
19118        runner
19119            .guarded_pr_message(&winner, None, true)
19120            .await
19121            .unwrap()
19122            .expect("rewritten");
19123        assert_eq!(runner.state.seats["impl-A"].turns, 2);
19124        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19125    }
19126
19127    #[tokio::test]
19128    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19129        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
19130        // session, was out of quota, and nothing else was asked.
19131        crate::run::pin_test_home();
19132        let dir = tempfile::tempdir().unwrap();
19133        let mut exhausted = spec("alpha");
19134        exhausted.command = vec!["false".into()];
19135        let mut runner = rewrite_runner(
19136            dir.path(),
19137            false,
19138            vec![exhausted, scripted("beta", EN_REWRITE)],
19139        );
19140        // The implementer's own agent is in the role too: it is not asked twice.
19141        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19142            "alpha".into(),
19143            "beta".into(),
19144        ]));
19145        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19146        seat.turns = 1;
19147        runner.state.seats.insert("impl-A".into(), seat);
19148        let winner = runner.state.candidates[0].clone();
19149        let msg = runner
19150            .guarded_pr_message(&winner, None, true)
19151            .await
19152            .unwrap()
19153            .expect("the role rewrites after the implementer fails");
19154        assert!(msg.body.contains("`src/net.rs`"));
19155        assert_eq!(
19156            runner.state.seats["impl-A"].turns, 2,
19157            "the implementer was tried once"
19158        );
19159        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19160        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19161    }
19162
19163    #[tokio::test]
19164    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19165        crate::run::pin_test_home();
19166        let dir = tempfile::tempdir().unwrap();
19167        let mut broken = spec("bad");
19168        broken.command = vec!["false".into()];
19169        let mut runner = rewrite_runner(
19170            dir.path(),
19171            true,
19172            vec![
19173                broken,
19174                scripted("noformat", "Add retries."),
19175                scripted("good", EN_REWRITE),
19176            ],
19177        );
19178        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19179            "bad".into(),
19180            "noformat".into(),
19181            "good".into(),
19182        ]));
19183        let winner = runner.state.candidates[0].clone();
19184        let msg = runner
19185            .guarded_pr_message(&winner, None, true)
19186            .await
19187            .unwrap()
19188            .expect("the third agent rewrites");
19189        assert!(msg.body.contains("`src/net.rs`"));
19190        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19191        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19192    }
19193
19194    #[tokio::test]
19195    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19196        crate::run::pin_test_home();
19197        // (agents, role): still Japanese; no TITLE; nothing resolvable.
19198        for (agents, role, saved) in [
19199            (
19200                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19201                Some("beta"),
19202                true,
19203            ),
19204            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19205            (Vec::new(), None, false),
19206        ] {
19207            let dir = tempfile::tempdir().unwrap();
19208            let mut runner = rewrite_runner(dir.path(), true, agents);
19209            runner.state.config.roles.language_judge =
19210                role.map(|r| crate::config::AgentChoice::One(r.into()));
19211            let winner = runner.state.candidates[0].clone();
19212            let first = runner
19213                .guarded_pr_message(&winner, None, true)
19214                .await
19215                .unwrap();
19216            assert!(first.is_none(), "withheld text parks the run");
19217            assert!(runner.state.parked);
19218            assert_eq!(gate_questions(&runner), 1);
19219            let g = runner.state.github_text.as_ref().unwrap();
19220            assert_eq!(g.rewrite.is_some(), saved);
19221            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
19222        }
19223    }
19224
19225    #[tokio::test]
19226    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
19227        crate::run::pin_test_home();
19228        let dir = tempfile::tempdir().unwrap();
19229        let mut runner = runner_at(RunStatus::Gating);
19230        runner.state =
19231            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19232        runner.state.repo = dir.path().to_owned();
19233        runner.state.candidates[0].worktree = dir.path().to_owned();
19234        let mut author = spec("alpha");
19235        author.command = vec![
19236            "sh".into(),
19237            "-c".into(),
19238            "printf '%s' \"$REWRITE_REPLY\"".into(),
19239        ];
19240        author.env.insert(
19241            "REWRITE_REPLY".into(),
19242            format!(
19243                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19244                "English sentence. ".repeat(600)
19245            ),
19246        );
19247        runner.state.config.agents = vec![author];
19248        seed_public_summary(&mut runner);
19249        let winner = runner.state.candidates[0].clone();
19250        assert!(
19251            runner
19252                .guarded_pr_message(&winner, None, true)
19253                .await
19254                .unwrap()
19255                .is_none()
19256        );
19257        let record = runner.state.github_text.clone().unwrap();
19258        let rewrite = record.rewrite.clone().unwrap();
19259        assert_eq!(rewrite.failed, ["body-language"]);
19260        assert!(
19261            rewrite
19262                .body
19263                .ends_with(&format!("magi:run/{}", runner.state.id))
19264        );
19265        let store = ask::Questions::open();
19266        let q = store.get(record.question.as_ref().unwrap()).unwrap();
19267        assert!(
19268            q.choices
19269                .contains(&crate::github_text::USE_REWRITE.to_owned())
19270        );
19271        for label in [
19272            "ORIGINAL",
19273            "FALLBACK",
19274            "REWRITE",
19275            "Rules still failed: body-language",
19276            "truncated",
19277        ] {
19278            assert!(q.detail.contains(label), "{label}");
19279        }
19280        let artifact = std::fs::read_to_string(crate::run::artifact_path(
19281            &runner.state,
19282            &crate::github_text::artifact_name(&record.fingerprint),
19283        ))
19284        .unwrap();
19285        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19286        // Restart from serialized state, then retry after a bad owner title.
19287        runner.state =
19288            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19289        store
19290            .update(&q.id, |q| {
19291                q.say("まだ日本語")?;
19292                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19293            })
19294            .unwrap();
19295        assert!(
19296            runner
19297                .guarded_pr_message(&winner, None, true)
19298                .await
19299                .unwrap()
19300                .is_none()
19301        );
19302        let retry = runner.state.github_text.as_ref().unwrap();
19303        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19304        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19305        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19306        assert!(
19307            q.posting_comparison
19308                .as_ref()
19309                .unwrap()
19310                .matches_detail(&q.detail)
19311        );
19312        store
19313            .update(&q.id, |q| {
19314                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19315            })
19316            .unwrap();
19317        let message = runner
19318            .guarded_pr_message(&winner, None, true)
19319            .await
19320            .unwrap()
19321            .unwrap();
19322        assert_eq!(message.title, rewrite.title);
19323        assert_eq!(message.body, rewrite.body);
19324        assert_eq!(
19325            runner.state.github_text.as_ref().unwrap().choice,
19326            Some(GithubTextChoice::Rewrite)
19327        );
19328        let again = runner
19329            .guarded_pr_message(&winner, None, true)
19330            .await
19331            .unwrap()
19332            .unwrap();
19333        assert_eq!(again.title, message.title);
19334        assert_eq!(again.body, message.body);
19335        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19336        assert!(
19337            !runner
19338                .state
19339                .events
19340                .iter()
19341                .any(|e| e.message.contains(&rewrite.title)
19342                    || e.message.contains("English sentence."))
19343        );
19344    }
19345
19346    #[tokio::test]
19347    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19348        crate::run::pin_test_home();
19349        let dir = tempfile::tempdir().unwrap();
19350        let mut runner = runner_at(RunStatus::Gating);
19351        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19352        runner.state.repo = dir.path().to_owned();
19353        runner.state.candidates[0].worktree = dir.path().to_owned();
19354        seed_public_summary(&mut runner);
19355        let winner = runner.state.candidates[0].clone();
19356        assert!(
19357            runner
19358                .guarded_pr_message(&winner, None, true)
19359                .await
19360                .unwrap()
19361                .is_none()
19362        );
19363        let record = runner.state.github_text.as_ref().unwrap();
19364        assert!(record.rewrite.is_none());
19365        // Records written before the additive field deserialize with no rewrite.
19366        let mut old = serde_json::to_value(record).unwrap();
19367        old.as_object_mut().unwrap().remove("rewrite");
19368        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19369        assert!(old.rewrite.is_none());
19370        let id = record.question.clone().unwrap();
19371        ask::Questions::open()
19372            .update(&id, |q| {
19373                q.choices.push(crate::github_text::USE_REWRITE.into());
19374                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19375            })
19376            .unwrap();
19377        let message = runner
19378            .guarded_pr_message(&winner, None, true)
19379            .await
19380            .unwrap()
19381            .unwrap();
19382        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19383        assert_eq!(
19384            runner.state.github_text.as_ref().unwrap().choice,
19385            Some(GithubTextChoice::Fallback)
19386        );
19387    }
19388
19389    #[tokio::test]
19390    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19391        crate::run::pin_test_home();
19392        let dir = tempfile::tempdir().unwrap();
19393        let mut runner = runner_at(RunStatus::Gating);
19394        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19395        runner.state.repo = dir.path().to_owned();
19396        runner.state.candidates[0].worktree = dir.path().to_owned();
19397        seed_public_summary(&mut runner);
19398        let winner = runner.state.candidates[0].clone();
19399        assert!(
19400            runner
19401                .guarded_pr_message(&winner, None, true)
19402                .await
19403                .unwrap()
19404                .is_none()
19405        );
19406        let store = ask::Questions::open();
19407        let find = |run: &str| {
19408            store
19409                .list()
19410                .into_iter()
19411                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19412                .unwrap()
19413        };
19414        let run = runner.state.id.clone();
19415        let q = find(&run);
19416        let fp = runner
19417            .state
19418            .github_text
19419            .as_ref()
19420            .unwrap()
19421            .fingerprint
19422            .clone();
19423        let name = crate::github_text::artifact_name(&fp);
19424        let path = crate::run::artifact_path(&runner.state, &name);
19425        assert!(q.detail.contains(&path.display().to_string()));
19426        assert!(q.detail.contains("title-language: title"));
19427        let saved = std::fs::read_to_string(&path).expect("artifact written");
19428        assert!(saved.contains("日本語のタイトル"));
19429        assert!(q.cwd.is_none());
19430        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19431        // A bad replacement is refused and asked about once more.
19432        let say = |id: &str, text: &str, choice: &str| {
19433            store
19434                .update(id, |q| {
19435                    q.thread.push(ask::Turn {
19436                        who: ask::Who::Operator,
19437                        body: text.to_owned(),
19438                        at: jiff::Timestamp::now(),
19439                        note: None,
19440                    });
19441                    q.answer(ask::Answer::Choice(choice.to_owned()))
19442                })
19443                .unwrap();
19444        };
19445        say(&q.id, "まだ日本語", "use my text");
19446        assert!(
19447            runner
19448                .guarded_pr_message(&winner, None, true)
19449                .await
19450                .unwrap()
19451                .is_none()
19452        );
19453        let second = find(&run);
19454        assert_ne!(second.id, q.id);
19455        assert!(second.detail.contains("did not pass"));
19456        // The retry still compares the original text; the rejected
19457        // replacement is named by category only.
19458        assert!(second.detail.contains("日本語のタイトル"));
19459        assert!(!second.detail.contains("まだ日本語"));
19460        assert!(second.detail.contains(&path.display().to_string()));
19461        say(&second.id, "fix: retry failed requests", "use my text");
19462        let message = runner
19463            .guarded_pr_message(&winner, None, true)
19464            .await
19465            .unwrap()
19466            .expect("a vetted replacement is posted");
19467        assert_eq!(message.title, "fix: retry failed requests");
19468        assert_eq!(
19469            runner
19470                .state
19471                .github_text
19472                .as_ref()
19473                .unwrap()
19474                .chosen_title
19475                .as_deref(),
19476            Some("fix: retry failed requests")
19477        );
19478        assert!(runner.state.github_text.as_ref().unwrap().resolved);
19479        // Resumed again: the saved decision is applied, nothing is re-asked.
19480        let again = runner
19481            .guarded_pr_message(&winner, None, true)
19482            .await
19483            .unwrap()
19484            .unwrap();
19485        assert_eq!(again.title, message.title);
19486        assert_eq!(
19487            store
19488                .list()
19489                .into_iter()
19490                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19491                .count(),
19492            2
19493        );
19494    }
19495
19496    #[test]
19497    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19498        // What magi itself writes stays English under any configured language,
19499        // so a future localisation of these headings fails here. (The agents'
19500        // own text is also checked by the posting gate.)
19501        let mut state = state_with_summary(
19502            "add retries",
19503            "TITLE: fix(web): batch reads\n- reads run.json once",
19504        );
19505        state.config.graph.language = "ja".to_owned();
19506        let m = pr_message(&state, 'A');
19507        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19508        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19509
19510        // The task is the operator's own text: it goes in untouched, and the
19511        // fallback title (no summary) may be in its language too.
19512        let task = "今回やってほしいこと: results projector を直す";
19513        let mut state = state_with_summary(task, "- no title line");
19514        state.config.graph.language = "ja".to_owned();
19515        let m = pr_message(&state, 'A');
19516        assert_eq!(
19517            m.title,
19518            format!("chore: land candidate A of run {}", state.id)
19519        );
19520        assert!(
19521            m.body.contains(&format!(
19522                "<summary>Original task</summary>\n\n{task}\n\n</details>"
19523            )),
19524            "{}",
19525            m.body
19526        );
19527    }
19528
19529    #[test]
19530    fn pr_message_scrubs_home_paths_and_addresses() {
19531        let state = state_with_summary(
19532            "fix it in /Users/someone/src/x",
19533            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19534        );
19535        let m = pr_message(&state, 'A');
19536        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19537            assert!(!m.body.contains(leak), "{}", m.body);
19538        }
19539        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19540    }
19541
19542    #[test]
19543    fn pr_message_survives_a_task_that_closes_details() {
19544        let state = state_with_summary("a </details> b", "TITLE: fix: x");
19545        let m = pr_message(&state, 'A');
19546        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19547    }
19548
19549    #[test]
19550    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19551        let cmd = manual_merge_command(
19552            MergeStyle::Squash,
19553            Path::new("/repo"),
19554            "b",
19555            "fix: \"quoted\" $(x) `y`\n\nbody",
19556        );
19557        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19558    }
19559
19560    #[test]
19561    fn manual_merge_command_matches_the_configured_style() {
19562        let repo = Path::new("/repo");
19563        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19564
19565        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19566        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19567
19568        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19569        assert_eq!(
19570            squash,
19571            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19572             \"Merge magi run 0832 (candidate A)\""
19573        );
19574
19575        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19576        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19577    }
19578
19579    #[test]
19580    fn a_nudge_gets_a_quarter_of_the_budget() {
19581        // The judge and implement budgets magi ships with.
19582        assert_eq!(retry_budget(secs(1200), true), secs(300));
19583        assert_eq!(retry_budget(secs(3600), true), secs(900));
19584    }
19585
19586    #[test]
19587    fn a_resent_prompt_keeps_the_whole_budget() {
19588        // The seat kept no context, so the retry is the original job again and
19589        // shortening it would only guarantee a second failure.
19590        assert_eq!(retry_budget(secs(1200), false), secs(1200));
19591        assert_eq!(retry_budget(secs(60), false), secs(60));
19592    }
19593
19594    #[test]
19595    fn the_floor_never_exceeds_the_original_budget() {
19596        // A short configured timeout must not be *raised* by the floor: the
19597        // operator asked for a bound, and a retry may not outlast the attempt
19598        // it is retrying.
19599        assert_eq!(retry_budget(secs(60), true), secs(60));
19600        assert_eq!(retry_budget(secs(480), true), secs(120));
19601        assert_eq!(retry_budget(secs(0), true), secs(0));
19602    }
19603
19604    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19605        agent::CommandEvidence {
19606            id: "item1".to_owned(),
19607            description: "cargo test".to_owned(),
19608            exit_code,
19609            result_summary: String::new(),
19610            source: "codex".to_owned(),
19611        }
19612    }
19613
19614    #[test]
19615    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19616        // No evidence is not the same fact as unconfirmed evidence: a
19617        // backend with no adapter, or a reply that ran no commands at all,
19618        // must not be misread as carrying a dangling job.
19619        assert!(!has_unconfirmed_command(&[]));
19620    }
19621
19622    #[test]
19623    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19624        // Deliberately not a check on the exit code's *value*: a fixer
19625        // legitimately runs something that fails mid-iteration before it
19626        // succeeds, and that must never by itself reopen a valid report.
19627        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19628        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19629        assert!(!has_unconfirmed_command(&[
19630            evidence(Some(0)),
19631            evidence(Some(101))
19632        ]));
19633    }
19634
19635    #[test]
19636    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19637        assert!(has_unconfirmed_command(&[
19638            evidence(Some(0)),
19639            evidence(None)
19640        ]));
19641    }
19642
19643    #[test]
19644    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19645        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19646        assert_eq!(
19647            verified_noop_claim(true, &[], text).as_deref(),
19648            Some("already fixed by b32cfc4, on main.")
19649        );
19650    }
19651
19652    #[test]
19653    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19654        // A timeout or a bad exit code reads as the ordinary loss it is,
19655        // whatever the reply's own prose claims.
19656        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19657        assert!(verified_noop_claim(false, &[], text).is_none());
19658    }
19659
19660    #[test]
19661    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19662        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19663        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19664        // A confirmed command alongside the marker is fine.
19665        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19666    }
19667
19668    #[test]
19669    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19670        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19671    }
19672
19673    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
19674    /// pair, in order, labelled A, B, C, ...
19675    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19676        runner.state.candidates = shape
19677            .iter()
19678            .enumerate()
19679            .map(|(i, &(empty, verified))| Candidate {
19680                index: i,
19681                label: (b'A' + i as u8) as char,
19682                agent: "sonnet".to_owned(),
19683                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19684                worktree: PathBuf::from(format!("/wt/{i}")),
19685                summary: String::new(),
19686                stat: String::new(),
19687                files: 0,
19688                commits: 0,
19689                empty,
19690                failed: None,
19691                verified_noop: verified.map(str::to_owned),
19692                duration_ms: 0,
19693                folded: false,
19694            })
19695            .collect();
19696    }
19697
19698    #[test]
19699    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19700        ask_test_home();
19701        let mut runner = runner_at(RunStatus::Implementing);
19702        set_candidates(
19703            &mut runner,
19704            &[
19705                (true, Some("already on main at b32cfc4")),
19706                (true, Some("same fix, see the existing test")),
19707            ],
19708        );
19709
19710        runner
19711            .after_implement()
19712            .expect("a verified no-op is not an error");
19713
19714        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19715    }
19716
19717    #[test]
19718    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19719        ask_test_home();
19720        let mut runner = runner_at(RunStatus::Implementing);
19721        // Candidate A declares a verified no-op; candidate B simply wrote
19722        // nothing and said nothing about why. One candidate's claim is not
19723        // the whole run's agreement.
19724        set_candidates(
19725            &mut runner,
19726            &[(true, Some("already on main at b32cfc4")), (true, None)],
19727        );
19728
19729        let err = runner
19730            .after_implement()
19731            .expect_err("an unverified empty candidate must still fail the run");
19732
19733        assert!(
19734            err.to_string().contains("no candidate produced a change"),
19735            "{err}"
19736        );
19737        assert_eq!(runner.state.status, RunStatus::Failed);
19738    }
19739
19740    #[test]
19741    fn after_implement_still_fails_an_ordinary_all_empty_run() {
19742        ask_test_home();
19743        let mut runner = runner_at(RunStatus::Implementing);
19744        set_candidates(&mut runner, &[(true, None), (true, None)]);
19745
19746        let err = runner
19747            .after_implement()
19748            .expect_err("no candidate declared anything; this is an ordinary failure");
19749
19750        assert!(
19751            err.to_string().contains("no candidate produced a change"),
19752            "{err}"
19753        );
19754        assert_eq!(runner.state.status, RunStatus::Failed);
19755    }
19756}
19757
19758/// A graft rollback whose restore could not be confirmed (see
19759/// [`Runner::graft`]).
19760#[derive(Debug)]
19761struct GraftRestoreFailed(String);
19762
19763impl std::fmt::Display for GraftRestoreFailed {
19764    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19765        write!(
19766            f,
19767            "could not restore the winner to its pre-graft commit: {}",
19768            self.0
19769        )
19770    }
19771}
19772
19773impl std::error::Error for GraftRestoreFailed {}
19774
19775/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
19776/// not the agent: they never draw on an agent's own timeout, and running out of
19777/// them never reads as an agent failure.
19778#[cfg(test)]
19779mod resource_wait_tests {
19780    use super::*;
19781    use std::collections::BTreeMap;
19782
19783    fn sh(id: &str, script: &str) -> AgentSpec {
19784        AgentSpec {
19785            id: id.to_owned(),
19786            kind: crate::config::AgentKind::Command,
19787            model: None,
19788            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19789            extra_args: Vec::new(),
19790            env: BTreeMap::new(),
19791            prompt_delivery: None,
19792            max_turn_secs: None,
19793        }
19794    }
19795
19796    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19797        SeatJob {
19798            seat: SeatState::new("impl-A", &spec.id, 1),
19799            spec,
19800            cwd: dir.to_path_buf(),
19801            prompt: "do it".to_owned(),
19802            timeout,
19803            allow_write: true,
19804            sessions: false,
19805            artifacts: dir.join("art"),
19806            stem: "impl-A".to_owned(),
19807            handover: None,
19808        }
19809    }
19810
19811    fn state(resource_wait_secs: u64) -> RunState {
19812        let mut config = Config::default();
19813        config.graph.timeout_resource_wait = resource_wait_secs;
19814        RunState::new(
19815            PathBuf::from("/repo"),
19816            "main".to_owned(),
19817            "abc1234".to_owned(),
19818            "task".to_owned(),
19819            config,
19820        )
19821    }
19822
19823    fn hold(cache: &Path) -> crate::cache::Guard {
19824        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19825        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19826            crate::cache::AcquireOutcome::Acquired(g) => g,
19827            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19828        }
19829    }
19830
19831    async fn run(
19832        jobs: Vec<SeatJob>,
19833        sem: Arc<Semaphore>,
19834        cache: Option<&Path>,
19835        st: &mut RunState,
19836    ) -> Vec<(usize, SeatState, AgentOutcome)> {
19837        let prompts = Prompts::default();
19838        let ctx = WaveCtx {
19839            run: "r",
19840            node: "implement",
19841            prompts: &prompts,
19842            cache,
19843            round: None,
19844            carry_seats: false,
19845            pauses: None,
19846        };
19847        wave(jobs, sem, &ctx, st, 0).await
19848    }
19849
19850    #[tokio::test]
19851    async fn verification_cache_wait_preserves_the_command_budget() {
19852        crate::run::pin_test_home();
19853        let tmp = tempfile::tempdir().unwrap();
19854        let cache = tmp.path().join("target");
19855        let guard = hold(&cache);
19856        let release = tokio::spawn(async move {
19857            tokio::time::sleep(Duration::from_millis(100)).await;
19858            guard.release();
19859        });
19860        let mut st = state(10);
19861        let budget = Duration::from_millis(50);
19862        let wait = repository_wait(&st, None);
19863        let (outcomes, _) = with_cache_lease(
19864            &mut st,
19865            Some(&cache),
19866            "gate",
19867            "gate",
19868            tmp.path(),
19869            "abc1234",
19870            budget,
19871            "test gate",
19872            &wait,
19873            |_, available| async move {
19874                assert_eq!(
19875                    available, budget,
19876                    "contention must not shorten verification"
19877                );
19878                (Vec::new(), false, Vec::new())
19879            },
19880        )
19881        .await
19882        .unwrap();
19883        release.await.unwrap();
19884        assert!(outcomes.is_empty(), "{outcomes:?}");
19885        assert!(
19886            st.events
19887                .iter()
19888                .any(|event| event.message.contains("waiting for"))
19889        );
19890        assert!(st.resource_wait.is_none());
19891        // The verification body no longer owns the lease after returning.
19892        hold(&cache).release();
19893    }
19894
19895    #[tokio::test]
19896    async fn verification_cache_wait_expires_without_starting_commands() {
19897        crate::run::pin_test_home();
19898        let tmp = tempfile::tempdir().unwrap();
19899        let cache = tmp.path().join("target");
19900        let _held = hold(&cache);
19901        let mut st = state(0);
19902        let wait = repository_wait(&st, None);
19903        let (outcomes, _) = with_cache_lease(
19904            &mut st,
19905            Some(&cache),
19906            "gate",
19907            "gate",
19908            tmp.path(),
19909            "abc1234",
19910            Duration::from_secs(60),
19911            "test gate",
19912            &wait,
19913            |_, _| async { panic!("commands must not start without the cache lease") },
19914        )
19915        .await
19916        .unwrap();
19917        assert_eq!(outcomes.len(), 1);
19918        assert!(outcomes[0].resource_blocked);
19919        assert!(st.resource_wait.is_none());
19920        assert!(matches!(
19921            st.resource_block.unwrap().cause,
19922            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19923        ));
19924    }
19925
19926    #[tokio::test]
19927    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19928        crate::run::pin_test_home();
19929        let tmp = tempfile::tempdir().unwrap();
19930        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19931        let cache = tmp.path().join("cache");
19932        let mut state = state(21600);
19933        let wait = repository_wait(&state, None);
19934        let (outcomes, _) = with_cache_lease(
19935            &mut state,
19936            Some(&cache),
19937            "gate",
19938            "gate",
19939            tmp.path(),
19940            "head",
19941            Duration::from_secs(60),
19942            "gate",
19943            &wait,
19944            |_, _| async { panic!("freshness failure must not start verification") },
19945        )
19946        .await
19947        .unwrap();
19948        assert_eq!(outcomes.len(), 1);
19949        assert!(outcomes[0].resource_blocked);
19950        let block = state.resource_block.unwrap();
19951        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19952        assert!(!block.description().contains("timed out"));
19953    }
19954
19955    #[tokio::test]
19956    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19957        crate::run::pin_test_home();
19958        let tmp = tempfile::tempdir().unwrap();
19959        let cache = tmp.path().join("cache");
19960        let _held = hold(&cache);
19961        let mut state = state(21600);
19962        let pause = Pause::new();
19963        let interrupt = Pause::new();
19964        let cause = ParkCause::Operator;
19965        pause.park_for(cause.clone());
19966        let prompts = Prompts::default();
19967        let ctx = WaveCtx {
19968            run: "test",
19969            node: "fix",
19970            prompts: &prompts,
19971            cache: Some(&cache),
19972            round: None,
19973            carry_seats: false,
19974            pauses: Some((&pause, &interrupt)),
19975        };
19976        let result = wave(
19977            vec![job(
19978                sh("writer", "exit 99"),
19979                tmp.path(),
19980                Duration::from_secs(1),
19981            )],
19982            Arc::new(Semaphore::new(1)),
19983            &ctx,
19984            &mut state,
19985            0,
19986        )
19987        .await;
19988        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19989        assert_eq!(result[0].1.turns, 0);
19990        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19991        assert_eq!(
19992            state.resource_block.unwrap().cause,
19993            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19994        );
19995        assert!(state.active.is_empty());
19996    }
19997
19998    #[tokio::test]
19999    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
20000        crate::run::pin_test_home();
20001        let tmp = tempfile::tempdir().unwrap();
20002        let cache = tmp.path().join("target-a");
20003        let guard = hold(&cache);
20004        // Held for longer than the agent's whole 1s budget.
20005        tokio::spawn(async move {
20006            tokio::time::sleep(Duration::from_millis(1500)).await;
20007            guard.release();
20008        });
20009        let mut st = state(60);
20010        let results = run(
20011            vec![job(
20012                sh("alpha", "echo ok"),
20013                tmp.path(),
20014                Duration::from_secs(1),
20015            )],
20016            Arc::new(Semaphore::new(1)),
20017            Some(&cache),
20018            &mut st,
20019        )
20020        .await;
20021        assert!(
20022            matches!(results[0].2, AgentOutcome::Ok(_)),
20023            "the wait must not be charged to the agent"
20024        );
20025        assert!(st.resource_block.is_none());
20026        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20027        assert!(st.resource_wait.is_none() && st.active.is_empty());
20028    }
20029
20030    #[tokio::test]
20031    #[cfg(unix)]
20032    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20033        use std::os::fd::AsRawFd as _;
20034        crate::run::pin_test_home();
20035        let tmp = tempfile::tempdir().unwrap();
20036        crate::git::git(tmp.path(), &["init"]).await.unwrap();
20037        let common = tmp.path().join(".git");
20038        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20039        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20040        assert_eq!(
20041            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20042            0
20043        );
20044        let mut st = state(1);
20045        st.config.coordination = crate::config::test_coordination();
20046        let mut reader = job(
20047            sh("reader", "echo must-not-run"),
20048            tmp.path(),
20049            Duration::from_secs(30),
20050        );
20051        reader.allow_write = false;
20052        let results = run(
20053            vec![reader.clone()],
20054            Arc::new(Semaphore::new(1)),
20055            None,
20056            &mut st,
20057        )
20058        .await;
20059        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20060        assert_eq!(results[0].1.turns, 0);
20061        assert!(FailClass::of(&results[0].2).is_none());
20062        assert_eq!(
20063            st.resource_block.as_ref().unwrap().kind,
20064            "repository coordination"
20065        );
20066        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20067        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20068        drop(external);
20069        let editor = crate::repo_lock::SeatLease::for_checkout(
20070            tmp.path(),
20071            true,
20072            crate::config::test_coordination().as_ref(),
20073        )
20074        .await
20075        .unwrap();
20076        let pause = Pause::new();
20077        let interrupt = Pause::new();
20078        let prompts = Prompts::default();
20079        let ctx = WaveCtx {
20080            run: "r",
20081            node: "review",
20082            prompts: &prompts,
20083            cache: None,
20084            round: None,
20085            carry_seats: false,
20086            pauses: Some((&pause, &interrupt)),
20087        };
20088        let (results, ()) = tokio::join!(
20089            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20090            async {
20091                tokio::time::sleep(Duration::from_millis(50)).await;
20092                pause.park();
20093            }
20094        );
20095        assert!(
20096            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20097        );
20098        assert_eq!(results[0].1.turns, 0);
20099        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20100        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20101        drop(editor);
20102    }
20103
20104    #[tokio::test]
20105    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20106        crate::run::pin_test_home();
20107        let tmp = tempfile::tempdir().unwrap();
20108        let cache = tmp.path().join("target-b");
20109        let _held = hold(&cache);
20110        let mut st = state(1);
20111        let results = run(
20112            vec![job(
20113                sh("alpha", "echo ok"),
20114                tmp.path(),
20115                Duration::from_secs(1),
20116            )],
20117            Arc::new(Semaphore::new(1)),
20118            Some(&cache),
20119            &mut st,
20120        )
20121        .await;
20122        let (_, seat, out) = &results[0];
20123        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20124        assert!(wave_blocked(&results));
20125        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20126        assert_eq!(seat.turns, 0, "nothing was asked");
20127        assert!(
20128            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20129            "no agent was spawned"
20130        );
20131        assert!(FailClass::of(out).is_none(), "not a failure class");
20132        let block = st.resource_block.as_ref().expect("recorded for the resume");
20133        assert_eq!(block.kind, "shared build cache");
20134        assert!(matches!(
20135            block.cause,
20136            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20137        ));
20138        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20139        assert!(st.active.is_empty());
20140    }
20141
20142    #[tokio::test]
20143    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20144        crate::run::pin_test_home();
20145        let tmp = tempfile::tempdir().unwrap();
20146        let mut st = state(1);
20147        let results = run(
20148            vec![job(
20149                sh("alpha", "echo ok"),
20150                tmp.path(),
20151                Duration::from_secs(1),
20152            )],
20153            Arc::new(Semaphore::new(0)),
20154            None,
20155            &mut st,
20156        )
20157        .await;
20158        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20159        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20160        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20161    }
20162
20163    #[tokio::test]
20164    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20165        crate::run::pin_test_home();
20166        let tmp = tempfile::tempdir().unwrap();
20167        let mut st = state(60);
20168        let results = run(
20169            vec![job(
20170                sh("alpha", "sleep 30"),
20171                tmp.path(),
20172                Duration::from_secs(1),
20173            )],
20174            Arc::new(Semaphore::new(1)),
20175            None,
20176            &mut st,
20177        )
20178        .await;
20179        let out = &results[0].2;
20180        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20181        let class = FailClass::of(out).expect("a failure");
20182        assert_eq!(class, FailClass::Timeout);
20183        assert!(should_hand_over(None, &class), "existing handover policy");
20184        assert!(st.resource_block.is_none());
20185    }
20186}