Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's Git administration access, network and repository coordination \
526         before retrying; an unattended seat cannot approve those permissions"
527    )
528}
529
530/// Outcome of the shared fixer chain, before caller-owned stop policy.
531enum FixStep {
532    Resource,
533    Recorded { report_lost: bool },
534}
535
536/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
537/// on an early return or a panic.
538///
539/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
540/// manual `magi fix` invocations against the same run are otherwise
541/// invisible to each other and would race to remove and recreate the same
542/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
543/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
544/// which is only ever reclaimed later, out of band, by
545/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
546/// `magi fix` invocation is not necessarily running under either of those, so
547/// nothing would ever sweep a lock a killed or crashed process left behind.
548/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
549/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
550/// lease use: an unreadable or unparsable pid, or a liveness query the
551/// platform cannot answer, reads as alive and the lock is left in place.
552struct FixClaim {
553    path: PathBuf,
554}
555
556impl FixClaim {
557    fn acquire(dir: &Path) -> Result<Self> {
558        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559        let path = dir.join("fix.lock");
560        let retry = crate::lock_file::Retry::new(None);
561        match Self::create(&path, retry) {
562            Ok(claim) => Ok(claim),
563            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564                if Self::reclaim_if_dead(&path) {
565                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566                } else {
567                    bail!(
568                        "another repair or `magi fix` is already running for this run ({} exists)",
569                        path.display()
570                    )
571                }
572            }
573            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574        }
575    }
576
577    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578        let mut f = retry.create(path)?;
579        use std::io::Write as _;
580        // Read back by `reclaim_if_dead` on a later, stuck invocation.
581        writeln!(f, "{}", std::process::id())?;
582        Ok(Self {
583            path: path.to_owned(),
584        })
585    }
586
587    /// True if the lock named a process confirmed dead, in which case it was
588    /// also removed. Never true on an unreadable file, an unparsable pid, or
589    /// a liveness query the platform cannot answer — see this type's own doc.
590    fn reclaim_if_dead(path: &Path) -> bool {
591        let dead = std::fs::read_to_string(path)
592            .ok()
593            .and_then(|body| body.trim().parse::<u32>().ok())
594            .is_some_and(|pid| !crate::proc::pid_alive(pid));
595        dead && std::fs::remove_file(path).is_ok()
596    }
597}
598
599impl Drop for FixClaim {
600    fn drop(&mut self) {
601        let _ = std::fs::remove_file(&self.path);
602    }
603}
604
605/// Holds the mutable runner as well as its durable lease: cancellation must
606/// clear both copies of ownership, including state a caller may save later.
607struct DriverExecution<'a> {
608    runner: &'a mut Runner,
609    driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613    fn drop(&mut self) {
614        self.driver.release_local(&mut self.runner.state);
615    }
616}
617
618impl Runner {
619    /// Start a fresh run against `repo`.
620    pub async fn start(
621        repo: &Path,
622        instruction: String,
623        config: Config,
624        origin: Origin,
625    ) -> Result<Self> {
626        Self::start_naming(repo, instruction, "", config, origin).await
627    }
628
629    /// [`Runner::start`] for a queued task. Titles and instructions never
630    /// select implementation sources. The title argument is retained for callers.
631    pub async fn start_naming(
632        repo: &Path,
633        instruction: String,
634        _also_scan: &str,
635        config: Config,
636        origin: Origin,
637    ) -> Result<Self> {
638        let repo = git::toplevel(repo).await?;
639        let missing = agent::missing_programs(&config.agents);
640        if !missing.is_empty() {
641            bail!(
642                "these agent programs are not on PATH: {}. Fix the roster in \
643                 magi.toml or install them.",
644                missing.join(", ")
645            );
646        }
647        let base_branch =
648            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649                .await?;
650        let fetched_base = resolve_base(
651            &repo,
652            &base_branch,
653            &config.merge.remote,
654            config.coordination.as_ref(),
655        )
656        .await?;
657        let base_commit = fetched_base.commit.clone();
658        let roles = config.resolve_roles()?;
659        let max_parallel = config.graph.max_parallel.max(1);
660        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661        // Recorded before the first save, so a crash right after minting
662        // cannot leave a run with no origin. Legibility only: nothing reads it
663        // to decide anything.
664        state.fetched_base = Some(fetched_base);
665        state.origin = Some(origin);
666        state.reserve()?;
667        state.event("start", format!("run {} created", state.id));
668        state.save()?;
669        Ok(Self {
670            state,
671            roles,
672            sem: Arc::new(Semaphore::new(max_parallel)),
673            pause: Pause::new(),
674            interrupt: Pause::new(),
675        })
676    }
677
678    /// Open a review-only run against work that already exists on `branch`.
679    ///
680    /// The expensive half of the graph is the implement wave — measured at
681    /// 111 and 134 internal tool-loop turns on this repository, against a
682    /// handful for a judge or a reviewer. The cheap half is worth running on
683    /// hand-written work too, and there was no way to reach it.
684    ///
685    /// No new state and no schema change are needed: a run with **one** viable
686    /// candidate and a tally already decided degrades `execute` to exactly
687    /// review → gate → merge, because `judge` skips a single-candidate field,
688    /// `deliberate` has fewer than two first choices to reconcile, `vote`
689    /// returns early, `tally` is already present and `fold_losers` has no
690    /// losers. Resuming such a run therefore does the right thing as well.
691    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692        Self::review_taking_over(repo, branch, config, None, origin).await
693    }
694
695    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
696    /// at the same task still has `branch` checked out, its worktree is
697    /// released first if that is safe (see [`crate::handover`]), and the
698    /// review refuses with the reason if it is not. `None` is a hand-run
699    /// review: it has no earlier attempts, so only a worktree of a dead run
700    /// magi recorded itself can be released.
701    pub async fn review_taking_over(
702        repo: &Path,
703        branch: &str,
704        config: Config,
705        takeover: Option<crate::handover::Takeover>,
706        origin: Origin,
707    ) -> Result<Self> {
708        let repo = git::toplevel(repo).await?;
709        let missing = agent::missing_programs(&config.agents);
710        if !missing.is_empty() {
711            bail!(
712                "these agent programs are not on PATH: {}. Fix the roster in \
713                 magi.toml or install them.",
714                missing.join(", ")
715            );
716        }
717        let base_branch =
718            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719                .await?;
720        if base_branch == branch {
721            bail!("`{branch}` is the base branch; there is nothing to review against");
722        }
723        let fetched_base = resolve_base(
724            &repo,
725            &base_branch,
726            &config.merge.remote,
727            config.coordination.as_ref(),
728        )
729        .await?;
730        let base_commit = fetched_base.commit.clone();
731
732        let roles = config.resolve_roles()?;
733        let max_parallel = config.graph.max_parallel.max(1);
734        let mut state = RunState::new(
735            repo.clone(),
736            base_branch,
737            base_commit.clone(),
738            String::new(),
739            config,
740        );
741        state.fetched_base = Some(fetched_base);
742        state.origin = Some(origin);
743        state.reserve()?;
744
745        // Released before anything else touches the branch: a stale local
746        // branch is moved with `git branch -f`, which git refuses while an
747        // earlier attempt's worktree still has it checked out. Everything
748        // after this point that can fail puts the old run back.
749        // A hand-run review has no task, hence no earlier attempts, but a
750        // worktree of a dead run magi made may still be released.
751        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752            earlier: Vec::new(),
753            home: crate::run::home(),
754            choice: None,
755            task_instruction: None,
756        });
757        let wait = repository_wait(&state, None);
758        let coordination_config = state.config.coordination.clone();
759        let coordination =
760            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761                .await?;
762        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763        if let Some(released) = &released {
764            state.event(
765                "release",
766                format!(
767                    "took `{branch}` over from run {}: its worktree was released: {}",
768                    crate::run::short_of(&released.old_id),
769                    released.audit
770                ),
771            );
772        }
773        // The owner's answer to an earlier divergence question is applied
774        // here: after the release (git will not move a checked-out branch)
775        // and before the sync that would otherwise ask again.
776        if let Some(choice) = takeover.choice.as_ref()
777            && let Err(e) =
778                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779                    .await
780        {
781            if let Some(released) = &released {
782                released.restore(&repo, branch).await;
783            }
784            return Err(e.context("applying the owner's answer about the diverged branch"));
785        }
786        // An earlier attempt of this task that published the task head, and
787        // whose work is the branch under review, hands that head over: the
788        // review run updates the same pull request. Without one (a legacy
789        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
790        // own head and its own open pull request is adopted as before.
791        for id in takeover.earlier.iter().rev() {
792            if let Ok(old) = RunState::load_under(id, &takeover.home)
793                && let Some(head) = old.pr_head.clone()
794                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795            {
796                state.pr_head = Some(head);
797                break;
798            }
799        }
800        drop(coordination);
801        let opened = Self::open_review(
802            &repo,
803            branch,
804            state,
805            roles,
806            max_parallel,
807            base_commit,
808            &takeover,
809        )
810        .await;
811        if opened.is_err()
812            && let Some(released) = &released
813        {
814            let _coordination = crate::repo_lock::SeatLease::operation_with(
815                &repo,
816                coordination_config.as_ref(),
817                &wait,
818            )
819            .await?;
820            released.restore(&repo, branch).await;
821        }
822        opened
823    }
824
825    /// Recover exactly the operator-selected public implementation, without touching old sources.
826    pub async fn recover_pr(
827        repo: &Path,
828        task: &str,
829        selection: crate::pr_recovery::Selection,
830        mut config: Config,
831    ) -> Result<Self> {
832        selection.configure(&mut config)?;
833        selection.check_task(task, &selection.instruction)?;
834        let base_branch =
835            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836                .await?;
837        let fetched_base = resolve_base(
838            repo,
839            &base_branch,
840            &config.merge.remote,
841            config.coordination.as_ref(),
842        )
843        .await?;
844        let base_commit = fetched_base.commit.clone();
845        selection
846            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847            .await?;
848        Box::pin(selection.verify_source(repo)).await?;
849        if let Some(proof) = &selection.source_proof {
850            anyhow::ensure!(
851                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852                "source rebase base is outside the fetched landing base"
853            );
854        }
855        let roles = config.resolve_roles()?;
856        let max_parallel = config.graph.max_parallel.max(1);
857        let mut state = RunState::new(
858            repo.to_path_buf(),
859            base_branch,
860            base_commit.clone(),
861            selection.instruction.clone(),
862            config,
863        );
864        state.fetched_base = Some(fetched_base);
865        state.origin = Some(crate::run::Origin::queue(task));
866        state.pr_head = Some(selection.branch.clone());
867        state.pr_recovery = Some(selection.clone());
868        state.reserve()?;
869        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870            // Never leave a reserved record behind as a phantom owner of the head.
871            state.discard_reserved();
872            return Err(e);
873        }
874        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875    }
876
877    async fn open_selected(
878        repo: &Path,
879        state: RunState,
880        roles: ResolvedRoles,
881        max_parallel: usize,
882        base_commit: String,
883        selection: crate::pr_recovery::Selection,
884    ) -> Result<Self> {
885        let mut state = state;
886        let branch = state.branch_for('A');
887        let wait = repository_wait(&state, None);
888        let prepared = async {
889            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890            git::git(repo, &["branch", &branch, selection.source()]).await?;
891            drop(coordination);
892            anyhow::Ok(())
893        }
894        .await;
895        if let Err(e) = prepared {
896            // Nothing was exposed yet: do not leave a reserved record as a
897            // phantom owner of the public head.
898            state.discard_reserved();
899            return Err(e);
900        }
901        state.event(
902            "recovery",
903            format!(
904                "selected implementation {} for {} (public lease {})",
905                selection.source(),
906                selection.url,
907                selection.head
908            ),
909        );
910        let takeover = crate::handover::Takeover {
911            earlier: Vec::new(),
912            home: crate::run::home(),
913            choice: None,
914            task_instruction: Some(selection.instruction.clone()),
915        };
916        if let Err(e) = state.save() {
917            state.discard_reserved();
918            return Err(e);
919        }
920        let id = state.id.clone();
921        let opened = Self::open_review(
922            repo,
923            &branch,
924            state,
925            roles,
926            max_parallel,
927            base_commit,
928            &takeover,
929        )
930        .await;
931        match opened {
932            Ok(runner) => Ok(runner),
933            Err(e) => {
934                if let Ok(mut partial) = RunState::load(&id) {
935                    partial.status = RunStatus::Blocked;
936                    partial.driver_exited = true;
937                    partial.event(
938                        "recovery",
939                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
940                    );
941                    partial.save()?;
942                }
943                Err(crate::pr_recovery::StartupFailed {
944                    run: id,
945                    cause: format!("{e:#}"),
946                }
947                .into())
948            }
949        }
950    }
951
952    /// What the newest earlier attempt that reviewed `branch` leaves for this
953    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
954    /// no earlier attempts. Reads the earlier record only: ownership of its
955    /// worktree and branch is [`crate::handover`]'s business.
956    async fn inherit(
957        repo: &Path,
958        branch: &str,
959        base_commit: &str,
960        takeover: &crate::handover::Takeover,
961    ) -> Option<crate::recovery::RecoveryRecord> {
962        use crate::recovery::{Relation, assess};
963        let newest = takeover.earlier.last()?;
964        let mut chosen = None;
965        for id in takeover.earlier.iter().rev() {
966            if let Ok(old) = RunState::load_under(id, &takeover.home)
967                && !old.reviews.is_empty()
968                && (old.winner().is_some_and(|w| w.branch == branch)
969                    || old.pr_head.as_deref() == Some(branch))
970            {
971                chosen = Some(old);
972                break;
973            }
974        }
975        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976            .await
977            .unwrap_or_default();
978        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979            Some(last) if last.head == tip => Relation::Same,
980            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981            _ => Relation::Diverged,
982        };
983        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984        Some(assess(from, chosen.as_ref(), base_commit, relation))
985    }
986
987    /// The half of [`Runner::review_taking_over`] that can fail after an
988    /// earlier attempt's worktree was released.
989    async fn open_review(
990        repo: &Path,
991        branch: &str,
992        mut state: RunState,
993        roles: ResolvedRoles,
994        max_parallel: usize,
995        base_commit: String,
996        takeover: &crate::handover::Takeover,
997    ) -> Result<Self> {
998        let wait = repository_wait(&state, None);
999        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000        if state.pr_recovery.is_none() {
1001            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002        }
1003        // The commit subjects are the closest thing to a task statement that
1004        // existing work carries, and the reviewers are told as much.
1005        let start = review_base(
1006            repo,
1007            &state.config.merge.remote,
1008            &state.base_branch,
1009            &base_commit,
1010            branch,
1011        )
1012        .await;
1013        let log = git::log_oneline(repo, &start, branch)
1014            .await
1015            .unwrap_or_default();
1016        let log_text = if log.trim().is_empty() {
1017            "(no commit messages)"
1018        } else {
1019            log.trim()
1020        };
1021        let mut instruction = format!(
1022            "Review the work already on branch `{branch}`. There is no task \
1023             statement: what the change claims to do is whatever its commits \
1024             say.\n\n{log_text}"
1025        );
1026        // A retry of a task whose earlier attempt left reviews behind carries
1027        // that attempt's evidence, and the task's own statement with it: the
1028        // completion criteria are the task's, not whatever the commits claim.
1029        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030            if let Some(task) = takeover
1031                .task_instruction
1032                .as_deref()
1033                .filter(|t| !t.trim().is_empty())
1034                && rec.reviewed_head.is_some()
1035            {
1036                instruction = format!(
1037                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038                    task.trim()
1039                );
1040                rec.criteria_inherited = true;
1041            }
1042            state.event("recovery", rec.summary());
1043            state.recovery = Some(rec);
1044        }
1045        if let Some(selected) = &state.pr_recovery {
1046            instruction = selected.instruction.clone();
1047        }
1048        state.task_instruction = takeover.task_instruction.clone();
1049        state.instruction = instruction;
1050        state.reviewed_commits = Some(
1051            git::subjects(repo, &start, branch)
1052                .await
1053                .unwrap_or_default(),
1054        );
1055
1056        // A takeover must not forget a hold that a previous review observed,
1057        // even if repairs have since removed it from the branch.
1058        for id in &takeover.earlier {
1059            if let Ok(old) = RunState::load_under(id, &takeover.home)
1060                && old.repo == state.repo
1061                && (old.pr_head.as_deref() == Some(branch)
1062                    || old.winner().is_some_and(|w| w.branch == branch))
1063            {
1064                for hold in old.branch_owner_holds {
1065                    if !state.branch_owner_holds.contains(&hold) {
1066                        state.event(
1067                            "approval-policy",
1068                            format!(
1069                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070                                hold.control.key(),
1071                                hold.file,
1072                                hold.head
1073                            ),
1074                        );
1075                        state.branch_owner_holds.push(hold);
1076                    }
1077                }
1078            }
1079        }
1080        let reviewed_head = git::rev_parse(repo, branch).await?;
1081        state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083        // An attached worktree, so the fixer's commits land on the branch under
1084        // review rather than on a detached head nobody will look at again.
1085        let worktree = state.worktree_root().join("under-review");
1086        if let Some(parent) = worktree.parent() {
1087            tokio::fs::create_dir_all(parent).await.ok();
1088        }
1089        let path = worktree.to_string_lossy().to_string();
1090        git::git(repo, &["worktree", "add", &path, branch])
1091            .await
1092            .with_context(|| {
1093                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094            })?;
1095
1096        let setup = match ensure_setup_config(&mut state, repo).await {
1097            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098            Err(e) => Err(e),
1099        };
1100        if let Err(e) = setup {
1101            git::worktree_remove_clean(repo, &worktree).await.ok();
1102            return Err(e);
1103        }
1104        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105            .await
1106            .unwrap_or(0);
1107        if commits == 0 {
1108            git::worktree_remove_clean(repo, &worktree).await.ok();
1109            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110        }
1111        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112            .await
1113            .map(|f| f.len())
1114            .unwrap_or(0);
1115        if files == 0
1116            && let (Ok(head_tree), Ok(base_tree)) = (
1117                git::tree_of(&worktree, "HEAD").await,
1118                git::tree_of(&worktree, &base_commit).await,
1119            )
1120            && head_tree == base_tree
1121        {
1122            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123            git::worktree_remove_clean(repo, &worktree).await.ok();
1124            bail!(
1125                "`{branch}` at {} has a tree identical to base {}; this usually means \
1126                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127                 against `{}/{branch}`) rather than an empty change",
1128                short(&head),
1129                short(&base_commit),
1130                state.config.merge.remote
1131            );
1132        }
1133        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134            .await
1135            .unwrap_or_default();
1136
1137        state.candidates.push(Candidate {
1138            index: 0,
1139            label: 'A',
1140            // Not an agent id on purpose: nothing in the roster wrote this, and
1141            // the stats tables must not credit anyone with a win for it.
1142            agent: EXISTING_BRANCH.to_owned(),
1143            branch: branch.to_owned(),
1144            worktree,
1145            summary: String::new(),
1146            stat,
1147            files,
1148            commits,
1149            empty: false,
1150            failed: None,
1151            verified_noop: None,
1152            duration_ms: 0,
1153            folded: false,
1154        });
1155        state.tally = Some(Tally {
1156            first_choice: BTreeMap::from([('A', 0)]),
1157            borda: BTreeMap::new(),
1158            winner: 'A',
1159            rankings: 0,
1160            unanimous_initial: false,
1161            deliberated: false,
1162            changed_votes: 0,
1163            unanimous_final: false,
1164            tie_break: None,
1165            // No panel sat, so no quorum applies. Zero judges is the correct
1166            // number for work that never competed, and must not be reported as
1167            // a collapsed panel.
1168            judges: 0,
1169            present: 0,
1170            quorum: 0,
1171            met_quorum: true,
1172            uncontested: Some("review-only run: nothing competed".to_owned()),
1173        });
1174        state.status = RunStatus::Reviewing;
1175        state.prep_complete = Some(true);
1176        state.event(
1177            "start",
1178            format!(
1179                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180                state.id
1181            ),
1182        );
1183        state.save()?;
1184        Ok(Self {
1185            state,
1186            roles,
1187            sem: Arc::new(Semaphore::new(max_parallel)),
1188            pause: Pause::new(),
1189            interrupt: Pause::new(),
1190        })
1191    }
1192
1193    /// Reopen an existing run.
1194    pub fn resume(id: &str) -> Result<Self> {
1195        let state = RunState::load(id)?;
1196        crate::pr_head::ensure_owner(&state)?;
1197        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198            bail!(
1199                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200            );
1201        }
1202        if let Some(to) = &state.released_to {
1203            bail!(
1204                "run {} cannot be resumed: its worktree was released to run {}",
1205                state.short(),
1206                crate::run::short_of(to)
1207            );
1208        }
1209        let roles = state.config.resolve_roles()?;
1210        if let Some(id) = &state.config.graph.selected_implementer
1211            && state.candidates.iter().any(|c| c.agent != *id)
1212        {
1213            bail!(
1214                "replacement implementer pin does not match the saved candidates; task must stay held"
1215            );
1216        }
1217        let max_parallel = state.config.graph.max_parallel.max(1);
1218        Ok(Self {
1219            state,
1220            roles,
1221            sem: Arc::new(Semaphore::new(max_parallel)),
1222            pause: Pause::new(),
1223            interrupt: Pause::new(),
1224        })
1225    }
1226
1227    /// Walk the graph to a terminal state, skipping nodes already recorded.
1228    ///
1229    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1230    /// the phone - ends here, so this is the one place a run that ended
1231    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1232    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1233    pub async fn execute(&mut self) -> Result<()> {
1234        if let Some(selection) = &self.state.pr_recovery
1235            && (selection.source_proof.is_some() || selection.source_run.is_some())
1236        {
1237            Box::pin(selection.verify_source(&self.state.repo)).await?;
1238        }
1239        // Queue claims cover daemon work; this also serializes manual drivers
1240        // consuming a repair answer, including the new review and decision.
1241        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242            || self.state.land_approval.is_some()
1243            || self.state.merge_decision.is_some()
1244            || !self.state.merge_repairs.is_empty()
1245            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246        {
1247            let claim = FixClaim::acquire(&self.state.dir())?;
1248            let disk = RunState::load(&self.state.id)?;
1249            let checkpoint = |s: &RunState| {
1250                (
1251                    s.merge_repairs.len(),
1252                    s.merge_repairs
1253                        .last()
1254                        .map(|r| (r.question.clone(), r.phase)),
1255                    s.land_approval
1256                        .as_ref()
1257                        .map(|a| (a.question.clone(), a.head.clone())),
1258                )
1259            };
1260            if checkpoint(&disk) != checkpoint(&self.state) {
1261                bail!(
1262                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263                );
1264            }
1265            Some(claim)
1266        } else {
1267            None
1268        };
1269        let mut execution = self.claim_driver().await?;
1270        let runner = &mut *execution.runner;
1271        let wait = runner.repository_wait();
1272        // Keep the graph's large future off the callers' stack, including
1273        // standalone task attempts running on a default-sized test thread.
1274        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275        let result = match result {
1276            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277                record_repository_block(&mut runner.state, &e);
1278                runner.park_here()?;
1279                Ok(())
1280            }
1281            other => other,
1282        };
1283        execution.driver.finish(&mut runner.state).await?;
1284        let ended = if result.is_err() {
1285            Some({
1286                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287                if let Err(error) = &result {
1288                    notice.message = format!(
1289                        "Run {} stopped with an error: {}",
1290                        runner.state.id,
1291                        error.to_string().lines().next().unwrap_or("unknown error")
1292                    );
1293                }
1294                notice
1295            })
1296        } else {
1297            crate::notices::run_ended(&runner.state)
1298        };
1299        if let Some(notice) = ended {
1300            crate::notices::raise(notice);
1301        }
1302        result
1303    }
1304
1305    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307    }
1308
1309    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310        let wait = self.repository_wait();
1311        coordinate_operation(&mut self.state, cwd, &wait).await
1312    }
1313
1314    /// Validate ownership and persist the driver claim in one critical section.
1315    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316        let owner = match crate::pr_head::owner_lock_wait(
1317            &crate::run::home(),
1318            &self.state.id,
1319            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320        )
1321        .await
1322        {
1323            Ok(owner) => owner,
1324            Err(error) => {
1325                let detail = format!("worktree maintenance or ownership update: {error:#}");
1326                // Admission did not own run.json: retain this resource outcome
1327                // in memory for queue settlement without overwriting a concurrent
1328                // retirement or publication transaction.
1329                self.state.resource_block = Some(crate::run::ResourceBlock {
1330                    cause: crate::run::ResourceBlockCause::Other,
1331                    node: "start".into(),
1332                    kind: "worktree maintenance".into(),
1333                    detail: detail.clone(),
1334                    at: Timestamp::now(),
1335                });
1336                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337            }
1338        };
1339        let disk = RunState::load(&self.state.id)?;
1340        crate::pr_head::ensure_owner(&disk)?;
1341        let claimed =
1342            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343        // Only proven-live ownership refuses here. Unknown records (legacy runs
1344        // with no claim marker or process identity) fall through to the
1345        // exclusive execution lease taken just below, which is what actually
1346        // keeps a second driver out.
1347        anyhow::ensure!(
1348            disk.liveness(claimed) != crate::run::Liveness::Live,
1349            "run already has a live driver; wait for its execution to end"
1350        );
1351        if let Some(to) = &disk.released_to {
1352            bail!(
1353                "run {} cannot continue: its worktree was released to run {}",
1354                self.state.short(),
1355                crate::run::short_of(to)
1356            );
1357        }
1358        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359        let mut execution = DriverExecution {
1360            runner: self,
1361            driver,
1362        };
1363        // Declare after the execution guard: on any claim error this short
1364        // lock must drop before that guard reloads and releases ownership.
1365        let _owner = owner;
1366        let state = &mut execution.runner.state;
1367        // Moving again, so it is no longer parked. Set before the walk rather
1368        // than in `resume`, so every way of re-entering the graph clears it
1369        // and a card cannot claim a run is waiting to be resumed while the
1370        // agents are already working.
1371        state.parked = false;
1372        state.resource_wait = None;
1373        state.resource_block = None;
1374        // Any seat this state still lists as answering belongs to whatever
1375        // process last drove this run — this one included, if it crashed
1376        // mid-wave. Cleared and flushed immediately, before anything else
1377        // runs, so a resume can never show a seat as live when nothing is
1378        // asking it anything yet; the node that actually dispatches the next
1379        // wave repopulates it.
1380        state.clear_active();
1381        // Recorded in the same spot, and flushed together with the clear
1382        // above: this is the pid a reader checks (`RunState::liveness`) when
1383        // no daemon claim exists to answer "is a process still driving this
1384        // run" — a plain `magi run` / `magi review` typed into a terminal
1385        // claims nothing there. Always overwritten, never only-if-absent, so
1386        // a resumed run's stale pid from a previous, possibly-dead process
1387        // can never survive into this one's own report. Unlike
1388        // `clear_active`, this changes on every single `execute()` call, so
1389        // the save below is now unconditional rather than only-if-cleared.
1390        //
1391        // `driver_started_at` is recorded in the same breath, from this same
1392        // pid, so `liveness` can tell a live pid that is genuinely still us
1393        // apart from one the OS has since handed to an unrelated process —
1394        // see that field's own doc for why the pid alone is not enough.
1395        let pid = std::process::id();
1396        state.driver_pid = Some(pid);
1397        state.driver_started_at = crate::proc::process_started_at(pid);
1398        state.driver_exited = false;
1399        state.driver_claimed = Some(true);
1400        state.begin_pass();
1401        execution.driver.bind(state);
1402        state.save()?;
1403        execution.driver.record(state)?;
1404        Ok(execution)
1405    }
1406
1407    async fn execute_graph(&mut self) -> Result<()> {
1408        self.state.configure_recovery_resume()?;
1409        // Preservation may invalidate an old empty classification. Resolve it
1410        // before any resume shortcut can reuse a verdict or publish a branch.
1411        self.persist_candidates().await?;
1412        if self
1413            .state
1414            .merge_decision
1415            .as_ref()
1416            .is_some_and(|decision| !decision.closed)
1417            && self.state.status != RunStatus::Landing
1418        {
1419            if crate::merge_decision::resumable_hold(&self.state) {
1420                self.state.merge = Some(MergeOutcome {
1421                    mode: MergeMode::Pr,
1422                    ok: false,
1423                    detail: self
1424                        .state
1425                        .merge_decision
1426                        .as_ref()
1427                        .unwrap()
1428                        .target
1429                        .pr
1430                        .clone(),
1431                    empty: false,
1432                });
1433                self.run_land().await?;
1434                self.settle_questions();
1435                return Ok(());
1436            }
1437            bail!(
1438                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439            );
1440        }
1441        if self.state.config.graph.selected_implementer.is_some()
1442            && self.state.status == RunStatus::Blocked
1443            && self.state.tally.is_none()
1444            && self.state.candidate_preservation.is_empty()
1445            && self.state.interrupted_implementations.is_empty()
1446        {
1447            bail!(
1448                "selected implementation is held; explicitly release the owning task to retry it"
1449            );
1450        }
1451
1452        if self.state.merge_repairs.last().is_some_and(|r| {
1453            matches!(
1454                r.phase,
1455                crate::run::MergeRepairPhase::Requested
1456                    | crate::run::MergeRepairPhase::Fixing
1457                    | crate::run::MergeRepairPhase::Reviewing
1458            )
1459        }) {
1460            if self.repair_merge_review().await? {
1461                self.run_land().await?;
1462            }
1463            self.settle_questions();
1464            return Ok(());
1465        }
1466        // Publication already passed review and gate. Resume only that node:
1467        // syncing to a newer base here could change the verified commit.
1468        if self.state.pr_publication_pending {
1469            self.merge().await?;
1470            if self.resource_halt()? {
1471                return Ok(());
1472            }
1473            self.state.save()?;
1474            self.settle_questions();
1475            return Ok(());
1476        }
1477        // A run that already lost its quorum never resumes into the verdict
1478        // machinery: `deliberate` and `vote` would otherwise clobber the
1479        // stalled marker back to Voting and the run would keep going past a
1480        // verdict that is no longer trustworthy. Everything already recorded is
1481        // kept, so the run stays resumable (or foldable) for a human to pick up.
1482        //
1483        // On --resume the run gets one chance to repair itself: the seats a
1484        // rate limit took out are re-asked. If their quota has since reset and
1485        // the quorum is restored, the run picks up and finishes; otherwise it
1486        // stays stale and still-resumable for a later retry. If it does not
1487        // recover, the returned status stays `Stalled` and nothing was
1488        // clobbered (the recovery only mutates entries for the lost seats).
1489        if self.state.status == RunStatus::Stalled {
1490            if self.recover_stall().await? {
1491                self.finish_after_tally().await?;
1492            } else {
1493                // Still below quorum: persist the marker and stay resumable.
1494                self.state.save()?;
1495            }
1496            return Ok(());
1497        }
1498        // A run parked inside `land` - watching CI, mid fix-round, or
1499        // waiting on the owner's merge approval - resumes directly into it,
1500        // never back through `prep`. Everything before `merge` already
1501        // concluded; that is the only way `status` reaches `Landing` in the
1502        // first place. Re-walking `review_loop` first would also be actively
1503        // wrong: its own status recomputation (see its doc) treats any
1504        // clean round as reason to set `status` to `Gating`, which would
1505        // clobber this marker before `merge` ever ran, and this run would
1506        // never find its way back into `land` at all.
1507        if self.state.status == RunStatus::Landing
1508            || crate::handover::landing_checkpoint(&self.state)
1509        {
1510            self.state.land_sync_pending = true;
1511            self.state.save()?;
1512            self.run_land().await?;
1513            // `run_land` may have settled the run right here - CI came back
1514            // green and the PR merged, say - without ever passing back
1515            // through `merge`'s own trailing call. Whatever it left `status`
1516            // as is what this has to read.
1517            self.settle_questions();
1518            return Ok(());
1519        }
1520        // A run parked at `merge` on the owner's word about a withheld PR text
1521        // resumes straight into `merge`. Re-walking the tail would sync to the
1522        // base again (possibly rebasing, possibly spending the fixer) after the
1523        // gate had passed, and then push a tree nobody reviewed or gated.
1524        if self.state.github_text.is_some()
1525            && self.state.merge.is_none()
1526            && self.state.status == RunStatus::Gating
1527        {
1528            self.merge().await?;
1529            if self.resource_halt()? {
1530                return Ok(());
1531            }
1532            self.state.save()?;
1533            return Ok(());
1534        }
1535        if let Err(e) = self.prep().await {
1536            if e.is::<crate::repo_lock::ResourceBlocked>() {
1537                return Err(e);
1538            }
1539            self.state.status = RunStatus::Blocked;
1540            self.state
1541                .prep_failure
1542                .get_or_insert_with(|| format!("{e:#}"));
1543            self.state
1544                .event("prep", format!("preparation stopped: {e:#}"));
1545            self.state.save()?;
1546            return Err(e);
1547        }
1548        if self.park_here()? {
1549            return Ok(());
1550        }
1551        self.advise().await?;
1552        if self.park_here()? {
1553            return Ok(());
1554        }
1555        self.implement().await?;
1556        if self.park_here()? {
1557            return Ok(());
1558        }
1559        // `after_implement` already saved the state and settled any open
1560        // questions when it set this; nothing later in the graph has
1561        // anything to judge.
1562        if self.state.status == RunStatus::VerifiedNoop {
1563            return Ok(());
1564        }
1565        self.judge().await?;
1566        if self.park_here()? {
1567            return Ok(());
1568        }
1569        self.deliberate().await?;
1570        if self.park_here()? {
1571            return Ok(());
1572        }
1573        self.vote().await?;
1574        if self.park_here()? {
1575            return Ok(());
1576        }
1577        self.tally()?;
1578        // A verdict that lost its quorum is not trustworthy: do not review,
1579        // gate, or merge on it. Everything already done is kept, so the run
1580        // stays resumable (or foldable); the human can replace the agent that
1581        // ran out of quota and pick it up.
1582        if self.state.status == RunStatus::Stalled {
1583            // Persist the stalled marker now — the normal end-of-execute save
1584            // below is below this early return, and without it a resumed run
1585            // would reload a pre-tally status and keep going.
1586            self.state.save()?;
1587            return Ok(());
1588        }
1589        self.finish_after_tally().await?;
1590        Ok(())
1591    }
1592
1593    /// A wave could not start its agents because a shared resource stayed busy
1594    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1595    /// resumable. Unstarted seats have no result; completed peers retain
1596    /// their saved candidate state.
1597    fn stop_for_resource(&mut self) -> Result<()> {
1598        // Resource acquisition observes cancellation inside a node. Complete
1599        // the same durable park as every other node boundary before returning.
1600        if self.pause.parked() || self.interrupt.parked() {
1601            self.park_here()?;
1602            return Ok(());
1603        }
1604        if self.state.resource_block.is_some() {
1605            self.state.status = RunStatus::Blocked;
1606        }
1607        self.state.save()
1608    }
1609
1610    /// Stop walking the graph if a node ended on a resource block.
1611    fn resource_halt(&mut self) -> Result<bool> {
1612        if self.state.resource_block.is_none() {
1613            return Ok(false);
1614        }
1615        self.stop_for_resource()?;
1616        Ok(true)
1617    }
1618
1619    /// Park here if asked to, recording it in the run's own timeline.
1620    ///
1621    /// Returns whether the caller should stop walking the graph. The state is
1622    /// saved either way by the node that just finished; this adds the event so
1623    /// the operator's card says why a run that is neither finished nor moving
1624    /// is sitting where it is.
1625    fn park_here(&mut self) -> Result<bool> {
1626        let parking = self.pause.parked() || self.interrupt.parked();
1627        if !parking && self.state.resource_block.is_some() {
1628            self.stop_for_resource()?;
1629            return Ok(true);
1630        }
1631        // Either handle asking is enough - see `Pause`'s own doc for why
1632        // they are never the same one. `interrupt` is checked second so a
1633        // reason it carries is preferred in the message below over a plain
1634        // shutdown park racing it at the same boundary.
1635        if !self.pause.parked() && !self.interrupt.parked() {
1636            return Ok(false);
1637        }
1638        if self.state.parked {
1639            return Ok(true);
1640        }
1641        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642        if let Some(block) = &mut self.state.resource_block
1643            && matches!(
1644                block.cause,
1645                crate::run::ResourceBlockCause::Cancelled { .. }
1646            )
1647        {
1648            block.cause = crate::run::ResourceBlockCause::Cancelled {
1649                park: cause.clone(),
1650            };
1651        }
1652        let why = match cause.as_ref().map(ParkCause::label) {
1653            Some(reason) => format!(
1654                "parked after `{}` ({reason}) — resume to carry on from here",
1655                self.state.status.as_str()
1656            ),
1657            None => format!(
1658                "parked after `{}` — resume to carry on from here",
1659                self.state.status.as_str()
1660            ),
1661        };
1662        self.state.event("park", why);
1663        self.state
1664            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665        self.state.parked = true;
1666        self.state.save()?;
1667        Ok(true)
1668    }
1669
1670    /// Only a new explicit operator approval may retry failed preparation.
1671    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672        if self.state.prep_failure.is_none() {
1673            return Ok(());
1674        }
1675        let failed_at = self
1676            .state
1677            .events
1678            .iter()
1679            .rev()
1680            .find(|event| event.node == "prep")
1681            .map_or(self.state.updated_at, |event| event.at);
1682        anyhow::ensure!(
1683            approval_at > failed_at,
1684            "preparation retry requires a new explicit release after the failure"
1685        );
1686        self.state.prep_failure = None;
1687        self.state.prep_complete = Some(false);
1688        self.state
1689            .event("prep-retry", "operator approved retry of preparation");
1690        Ok(())
1691    }
1692
1693    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1694    pub fn on_pause(&mut self, pause: Pause) {
1695        self.pause = pause;
1696    }
1697
1698    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1699    /// scheduler asking this one run - and no other - to park so a task
1700    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1701    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1702    /// handle.
1703    pub fn watch_interrupt(&mut self, pause: Pause) {
1704        self.interrupt = pause;
1705    }
1706
1707    /// Abandon this run's own open questions, once `status` has actually
1708    /// settled rather than merely paused.
1709    ///
1710    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1711    /// either back up with the candidates, the review round and the seat
1712    /// sessions already on disk, so a question an implementer asked mid-round
1713    /// may still get a real answer read by a real resume. Only the statuses
1714    /// `resumable` excludes are actually final: the run merged, it reached
1715    /// `Ready` with nothing left to do, it failed outright with no
1716    /// established point to continue from, or every candidate agreed, with
1717    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1718    /// every one of those the seat that asked is gone for good, exactly like
1719    /// the run being deleted under `magi run rm` - so the same cleanup
1720    /// applies, worded for what actually happened instead of "the run was
1721    /// deleted".
1722    ///
1723    /// Best-effort and silent on success: called from every place `status`
1724    /// can land on one of those three, including ones a resumed run revisits,
1725    /// so it must cost nothing when there was nothing open to begin with.
1726    fn settle_questions(&mut self) {
1727        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729        }
1730    }
1731
1732    /// The tail of the graph after a trustworthy tally: fold losers, review,
1733    /// gate, merge, and persist.
1734    async fn finish_after_tally(&mut self) -> Result<()> {
1735        // Before the losers are folded: their branches are what the graft
1736        // reads from.
1737        self.graft().await?;
1738        if self.resource_halt()? {
1739            return Ok(());
1740        }
1741        self.fold_losers().await?;
1742        // Before review starts, and again right before the gate: a run's
1743        // review rounds can themselves take long enough for the base to move
1744        // a second time, and the gate is the one node whose "green" gets
1745        // acted on.
1746        self.sync_to_base().await?;
1747        if self.resource_halt()? {
1748            return Ok(());
1749        }
1750        if self.state.status == RunStatus::AlreadyInBase {
1751            return Ok(());
1752        }
1753        self.review_loop().await?;
1754        if self.resource_halt()? {
1755            return Ok(());
1756        }
1757        self.sync_to_base().await?;
1758        if self.resource_halt()? {
1759            return Ok(());
1760        }
1761        if self.state.status == RunStatus::AlreadyInBase {
1762            return Ok(());
1763        }
1764        self.gate().await?;
1765        self.merge().await?;
1766        if self.resource_halt()? {
1767            return Ok(());
1768        }
1769        self.state.save()?;
1770        Ok(())
1771    }
1772
1773    // ---------------------------------------------------------------- prep
1774
1775    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1776    /// run with the step and its output in the event; it is never swallowed.
1777    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1778    /// resume starts over instead of finding candidates and missing seats.
1779    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780        let result = match ensure_setup_config(&mut self.state, repo).await {
1781            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782            Err(e) => Err(e),
1783        };
1784        if let Err(e) = result {
1785            git::worktree_remove_clean(repo, wt).await.ok();
1786            if in_prep {
1787                self.rollback_prep(repo).await;
1788            }
1789            self.state.status = RunStatus::Blocked;
1790            self.state.event(
1791                "setup",
1792                format!("worktree setup failed in {}: {e:#}", wt.display()),
1793            );
1794            self.state.save()?;
1795            return Err(e);
1796        }
1797        Ok(())
1798    }
1799
1800    /// Undo everything `prep` created, so a resume re-runs it whole.
1801    async fn rollback_prep(&mut self, repo: &Path) {
1802        let root = self.state.worktree_root();
1803        for c in std::mem::take(&mut self.state.candidates) {
1804            if matches!(
1805                git::worktree_remove_clean(repo, &c.worktree).await,
1806                Ok(true)
1807            ) {
1808                git::branch_delete(repo, &c.branch).await.ok();
1809            } else {
1810                self.state.candidates.push(c);
1811            }
1812        }
1813        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814        for j in 1..=seats {
1815            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816                .await
1817                .ok();
1818        }
1819        for k in 1..=self.state.config.graph.advisors {
1820            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821                .await
1822                .ok();
1823        }
1824    }
1825
1826    async fn prep(&mut self) -> Result<()> {
1827        if let Some(reason) = &self.state.prep_failure {
1828            bail!(
1829                "candidate preparation previously failed: {reason}; correct the source before retrying"
1830            );
1831        }
1832        if self
1833            .state
1834            .prep_complete
1835            .unwrap_or(!self.state.candidates.is_empty())
1836        {
1837            return Ok(());
1838        }
1839        let repo = self.state.repo.clone();
1840        // Includes rollback, config, refs, checkout creation and setup. No
1841        // repository mutation may precede the real shared lock acquisition.
1842        let _coordination = self.coordinate(&repo).await?;
1843        let base = self.state.base_commit.clone();
1844        let plan = self.state.sources.plan(&repo, &base).await?;
1845        self.rollback_prep(&repo).await;
1846        if !self.state.candidates.is_empty() {
1847            bail!(
1848                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849            );
1850        }
1851        self.state.prep_complete = Some(false);
1852        self.state.status = RunStatus::Prep;
1853        self.state.save()?;
1854        let repo = self.state.repo.clone();
1855        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856        let root = self.state.worktree_root();
1857        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859        // The hook is the write-time half of the blindness contract; the
1860        // presentation filter in `blind` is the half that cannot be bypassed.
1861        let hooks_dir = self.state.dir().join("hooks");
1862        if self.state.config.blind.commit_msg_hook {
1863            std::fs::create_dir_all(&hooks_dir)
1864                .with_context(|| format!("create {}", hooks_dir.display()))?;
1865            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866            let path = hooks_dir.join("commit-msg");
1867            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868            make_executable(&path)?;
1869            // Ref-counted rather than a plain idempotent set: with more than
1870            // one run able to be in flight in the same repository at once
1871            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1872            // true?" check cannot tell "another run of mine still needs
1873            // this" from "nobody does", and the run that happens to finish
1874            // first would disable the hook out from under a sibling still
1875            // relying on it.
1876            git::acquire_worktree_config(&repo).await?;
1877            self.state.enabled_worktree_config = true;
1878        }
1879
1880        // `[worktree] setup` hides its products per worktree, which needs the
1881        // same per-worktree config the hook does. Held until fold, like it.
1882        ensure_setup_config(&mut self.state, &repo).await?;
1883
1884        for (index, (spec, label)) in self
1885            .roles
1886            .implementers
1887            .clone()
1888            .into_iter()
1889            .zip(labels)
1890            .enumerate()
1891        {
1892            let branch = self.state.branch_for(label);
1893            let worktree = root.join(format!("cand-{label}"));
1894            // Persist intent before Git can leave a partially created tree.
1895            // Candidate registration follows setup, so it cannot inventory
1896            // crashes or refusals during creation itself.
1897            self.state.event(
1898                "prep-intent",
1899                serde_json::json!({
1900                    "branch": branch,
1901                    "worktree": worktree,
1902                    "start": start,
1903                    "picks": plan.picks,
1904                })
1905                .to_string(),
1906            );
1907            self.state.save()?;
1908            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909            if self.state.config.blind.commit_msg_hook {
1910                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911            }
1912            git::local_exclude(&worktree, "/.magi/").await?;
1913            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914                // Nothing of this prep may survive: a resume must start over,
1915                // not find half the candidates and skip the rest.
1916                git::worktree_remove_clean(&repo, &worktree).await.ok();
1917                git::branch_delete(&repo, &branch).await.ok();
1918                self.rollback_prep(&repo).await;
1919                self.state.status = RunStatus::Blocked;
1920                self.state
1921                    .event("prep", format!("worktree setup failed: {e:#}"));
1922                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923                self.state.save()?;
1924                return Err(e);
1925            }
1926            self.state.candidates.push(Candidate {
1927                index,
1928                label,
1929                agent: spec.id.clone(),
1930                branch,
1931                worktree,
1932                summary: String::new(),
1933                stat: String::new(),
1934                files: 0,
1935                commits: 0,
1936                empty: false,
1937                failed: None,
1938                verified_noop: None,
1939                duration_ms: 0,
1940                folded: false,
1941            });
1942            self.state.save()?;
1943            for pick in &plan.picks {
1944                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945                {
1946                    self.state.status = RunStatus::Blocked;
1947                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948                    self.state
1949                        .event("prep", format!("cannot apply referenced commit: {e}"));
1950                    self.state.save()?;
1951                    return Err(e);
1952                }
1953            }
1954        }
1955
1956        for j in 1..=self.roles.judges.len() {
1957            let wt = root.join(format!("judge-{j}"));
1958            if !wt.exists() {
1959                git::worktree_add_detached(&repo, &wt, &base).await?;
1960                self.setup_seat_worktree(&repo, &wt, true).await?;
1961            }
1962        }
1963
1964        // Disposable, detached checkouts for the design-deliberation stage's
1965        // advisor seats — the same shape as the judges' above, at the same
1966        // base commit, since advisors also only ever read. Sized off the
1967        // configured count directly rather than a resolved roster: unlike
1968        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1969        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1970        // `prep` has no `ResolvedRoles` field to read a count from here.
1971        if self.state.config.graph.advise {
1972            for k in 1..=self.state.config.graph.advisors {
1973                let wt = root.join(format!("advisor-{k}"));
1974                if !wt.exists() {
1975                    git::worktree_add_detached(&repo, &wt, &base).await?;
1976                    self.setup_seat_worktree(&repo, &wt, true).await?;
1977                }
1978            }
1979        }
1980
1981        // A judge cannot tell it is looking at its own patch — the seats keep
1982        // separate conversations — but a panel that shares agents with the
1983        // field is less independent than it looks, and that is worth saying out
1984        // loud once per run rather than leaving it in the config.
1985        let authors: Vec<&str> = self
1986            .roles
1987            .implementers
1988            .iter()
1989            .map(|a| a.id.as_str())
1990            .collect();
1991        let overlap: Vec<String> = self
1992            .roles
1993            .judges
1994            .iter()
1995            .enumerate()
1996            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998            .collect();
1999        if !overlap.is_empty() {
2000            let note = format!(
2001                "{} also authored a candidate; blind, but the panel is less \
2002                 independent than {} distinct agents would be",
2003                overlap.join(", "),
2004                self.roles.judges.len()
2005            );
2006            self.state.event("prep", note);
2007        }
2008
2009        self.state.event(
2010            "prep",
2011            format!(
2012                "{} candidates, {} judges, base {} ({})",
2013                self.state.candidates.len(),
2014                self.roles.judges.len(),
2015                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016                self.state.base_branch
2017            ),
2018        );
2019        self.state.prep_complete = Some(true);
2020        self.state.status = RunStatus::Implementing;
2021        self.state.save()?;
2022        Ok(())
2023    }
2024
2025    // -------------------------------------------------------------- advise
2026
2027    /// The design-deliberation stage: independent, read-only advisor seats
2028    /// each sketch a design before any implementer touches the repository,
2029    /// and (when at least one produced a usable proposal) a synthesis seat
2030    /// blends them into a brief `implement` carries in every candidate's
2031    /// prompt.
2032    ///
2033    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2034    /// advisors` is the proposal count. Everything here is best-effort and
2035    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2036    /// that cannot reach quota, or a synthesis seat that produced nothing
2037    /// usable all leave `implement` exactly as it was before this stage
2038    /// existed — the task instruction alone — rather than failing the whole
2039    /// competition over an enrichment stage. Every outcome is still recorded
2040    /// as an event, so a run that got nothing from this stage says why.
2041    ///
2042    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2043    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2044    /// resumed run whose stage failed would re-run it, and re-spend the
2045    /// agent calls, on every reentry before `implement`.
2046    ///
2047    /// Also skipped once any candidate shows implementation progress — the
2048    /// exact predicate `implement` itself uses to decide a candidate is no
2049    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2050    /// is not enough: a run created by an older binary that predates this
2051    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2052    /// an already-`Implementing`-or-later run under this build would
2053    /// otherwise walk straight back through `prep` (a no-op once candidates
2054    /// exist) into this node and spawn every advisor seat against worktrees
2055    /// `prep` never recreated — after implementation has already started,
2056    /// which is exactly the invariant this stage exists to guarantee.
2057    async fn advise(&mut self) -> Result<()> {
2058        let implement_untouched = self
2059            .state
2060            .candidates
2061            .iter()
2062            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063        if !self.state.config.graph.advise || self.state.advise_attempted {
2064            return Ok(());
2065        }
2066        if !implement_untouched {
2067            self.state.event(
2068                "advise",
2069                "skipping the design-deliberation stage: at least one \
2070                 candidate already shows implementation progress, so this \
2071                 run is past the point the stage exists to run before"
2072                    .to_owned(),
2073            );
2074            self.state.advise_attempted = true;
2075            self.state.save()?;
2076            return Ok(());
2077        }
2078        let run_id = self.state.id.clone();
2079        let prompts = self.state.config.prompts.clone();
2080        let instruction = self.state.instruction.clone();
2081        let language = self.state.config.graph.language.clone();
2082        let root = self.state.worktree_root();
2083        let n = self.state.config.graph.advisors;
2084        let where_recorded = self.state.dir().join("run.json");
2085
2086        let seats = match self.state.config.advisors() {
2087            Ok(seats) if !seats.is_empty() => seats,
2088            Ok(_) => {
2089                self.state.event(
2090                    "advise",
2091                    format!(
2092                        "[graph] advisors is 0; skipping the design-deliberation \
2093                         stage and continuing without a synthesis brief (see {})",
2094                        where_recorded.display()
2095                    ),
2096                );
2097                self.state.advise_attempted = true;
2098                self.state.save()?;
2099                return Ok(());
2100            }
2101            Err(e) => {
2102                self.state.event(
2103                    "advise",
2104                    format!(
2105                        "could not resolve advisor seats ({e:#}); continuing \
2106                         without a design-deliberation brief (see {})",
2107                        where_recorded.display()
2108                    ),
2109                );
2110                self.state.advise_attempted = true;
2111                self.state.save()?;
2112                return Ok(());
2113            }
2114        };
2115
2116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117        let artifacts = agent::artifacts_dir(&self.state.dir());
2118        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120        let mut jobs = Vec::new();
2121        for (i, spec) in seats.iter().cloned().enumerate() {
2122            let seat_key = format!("advisor-{}", i + 1);
2123            let seat = self.seat(&seat_key, &spec.id);
2124            jobs.push(SeatJob {
2125                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126                spec,
2127                seat,
2128                cwd: worktrees[i % worktrees.len()].clone(),
2129                timeout,
2130                allow_write: false,
2131                sessions: false,
2132                artifacts: artifacts.clone(),
2133                stem: seat_key,
2134                handover: None,
2135            });
2136        }
2137
2138        self.state.event(
2139            "advise",
2140            format!(
2141                "{} advisor seat(s) sketching a design in parallel",
2142                jobs.len()
2143            ),
2144        );
2145        let mut quota_losses = Vec::new();
2146        let cache = self.state.config.cache_dir();
2147        let ctx = WaveCtx {
2148            carry_seats: false,
2149            pauses: Some((&self.pause, &self.interrupt)),
2150            run: &run_id,
2151            node: "advise",
2152            prompts: &prompts,
2153            cache: cache.as_deref(),
2154            round: None,
2155        };
2156        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157        let results = ask_json_wave::<Proposal>(
2158            jobs,
2159            Arc::clone(&self.sem),
2160            self.state.config.graph.retries,
2161            &advisor_roster,
2162            &ctx,
2163            &mut quota_losses,
2164            &mut self.state,
2165            &|p: &Proposal| p.validate(),
2166        )
2167        .await;
2168        if self.state.resource_block.is_some() {
2169            return self.stop_for_resource();
2170        }
2171        self.state.quota.extend(quota_losses);
2172
2173        let mut records = Vec::with_capacity(results.len());
2174        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175            let agent_id = seat.agent.clone();
2176            self.state.seats.insert(seat.key.clone(), seat);
2177            match res {
2178                Ok((proposal, out)) => {
2179                    self.state
2180                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2181                    records.push(advise::AdvisorRecord::proposed(
2182                        i + 1,
2183                        agent_id,
2184                        proposal,
2185                        out.duration_ms,
2186                    ));
2187                }
2188                Err(e) => {
2189                    self.state.event(
2190                        "advise",
2191                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192                    );
2193                    records.push(advise::AdvisorRecord::failed(
2194                        i + 1,
2195                        agent_id,
2196                        e.to_string(),
2197                    ));
2198                }
2199            }
2200        }
2201
2202        let mut advice = advise::Advice {
2203            records,
2204            synthesis: None,
2205        };
2206        if advice.proposals().is_empty() {
2207            self.state.event(
2208                "advise",
2209                "no advisor produced a usable proposal; continuing without a \
2210                 synthesis brief"
2211                    .to_owned(),
2212            );
2213        } else {
2214            match self
2215                .synthesize_brief(
2216                    &advice,
2217                    &instruction,
2218                    &language,
2219                    &worktrees[0],
2220                    &artifacts,
2221                    &run_id,
2222                    &prompts,
2223                    cache.as_deref(),
2224                )
2225                .await
2226            {
2227                Ok(Some(text)) => {
2228                    self.state.event(
2229                        "advise",
2230                        "synthesized a design brief for the implementer".to_owned(),
2231                    );
2232                    advice.synthesis = Some(text);
2233                }
2234                Ok(None) => {
2235                    self.state.event(
2236                        "advise",
2237                        "the synthesis seat produced nothing usable; continuing \
2238                         without a design brief"
2239                            .to_owned(),
2240                    );
2241                }
2242                Err(e) => {
2243                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2244                        return Err(e);
2245                    }
2246                    self.state.event(
2247                        "advise",
2248                        format!("could not synthesize a design brief: {e:#}"),
2249                    );
2250                }
2251            }
2252        }
2253        advise::apply_reflection(&mut advice);
2254
2255        self.state.advice = Some(advice);
2256        self.state.advise_attempted = true;
2257        self.state.save()?;
2258        Ok(())
2259    }
2260
2261    /// The synthesis seat: reads every advisor's proposal and blends them
2262    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2263    /// out of [`Runner::advise`] only for readability — it is not called
2264    /// anywhere else.
2265    ///
2266    /// Picked the same way [`crate::talk`]'s standing conversation and
2267    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2268    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2269    /// order (a claude seat, else the first runnable agent in roster order)
2270    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2271    /// in [`crate::config`] for why a dedicated field exists here at all.
2272    #[allow(clippy::too_many_arguments)]
2273    async fn synthesize_brief(
2274        &mut self,
2275        advice: &advise::Advice,
2276        instruction: &str,
2277        language: &str,
2278        cwd: &Path,
2279        artifacts: &Path,
2280        run_id: &str,
2281        prompts: &Prompts,
2282        cache: Option<&Path>,
2283    ) -> Result<Option<String>> {
2284        let chain = agent::pick_chain(
2285            &self.state.config.agents,
2286            self.state.config.roles.synthesizer.as_ref(),
2287            &agent::installed,
2288            "synthesizer",
2289        )?;
2290        let proposals = advice.proposals();
2291        let mut prompt = prompt::with_overlay(
2292            prompt::synthesize_brief(instruction, &proposals, language),
2293            prompts.overlay("advise"),
2294        );
2295        if cache.is_some() {
2296            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2297            // below — see `prompt::build_cache_note`'s doc for why telling a
2298            // read-only seat to build through the shared cache is exactly how
2299            // a sandbox's write refusal gets misread as a defect.
2300            prompt.push('\n');
2301            prompt.push_str(&prompt::build_cache_note("advise", false));
2302        }
2303        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304        // Each id is tried once, in order; a quota hit, error or unusable
2305        // answer moves to the next. The seat is single-turn (`sessions:
2306        // false`) and the prompt is the whole context, so a fallback agent
2307        // needs nothing carried over.
2308        let mut last = None;
2309        for (n, spec) in chain.iter().enumerate() {
2310            if n > 0 {
2311                self.state
2312                    .event("advise", format!("synthesis falling back to {}", spec.id));
2313            }
2314            let mut seat = self.seat("advise-synthesis", &spec.id);
2315            let wait = self.repository_wait();
2316            let outcome = wait
2317                .scope(agent::invoke(
2318                    spec,
2319                    &mut seat,
2320                    &Invocation {
2321                        coordination: self.state.config.coordination.as_ref(),
2322                        editing_network: self.state.config.graph.editing_network,
2323                        fetched_base: None,
2324                        cwd,
2325                        prompt: &prompt,
2326                        timeout,
2327                        allow_write: false,
2328                        unsandboxed: false,
2329                        sessions: false,
2330                        artifacts,
2331                        stem: &if n == 0 {
2332                            "advise-synthesis".to_owned()
2333                        } else {
2334                            format!("advise-synthesis-{}", spec.id)
2335                        },
2336                        run: run_id,
2337                        node: "advise",
2338                        cache_dir: None,
2339                        attachments: &[],
2340                        writable: &[],
2341                    },
2342                ))
2343                .await;
2344            if outcome.is_ok() {
2345                self.state.seats.insert(seat.key.clone(), seat);
2346            }
2347            let advance = agent::chain_advances(&outcome);
2348            last = Some(outcome);
2349            if !advance {
2350                break;
2351            }
2352        }
2353        // Exhausted: the last attempt's result is what a single failed seat
2354        // would have produced.
2355        let out = last.expect("a chain holds at least one agent")?;
2356        if !out.usable() {
2357            return Ok(None);
2358        }
2359        let text =
2360            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2361        Ok((!text.trim().is_empty()).then_some(text))
2362    }
2363
2364    // ----------------------------------------------------------- implement
2365
2366    async fn implement(&mut self) -> Result<()> {
2367        self.persist_candidates().await?;
2368        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2369        // agent files with `magi task add` name the run that paid for it. The
2370        // prompt overlay is cloned alongside it because the waves borrow it
2371        // while `self` is mutably borrowed by the node's own bookkeeping.
2372        let run_id = self.state.id.clone();
2373        let prompts = self.state.config.prompts.clone();
2374        let todo: Vec<usize> = self
2375            .state
2376            .candidates
2377            .iter()
2378            .enumerate()
2379            .filter(|(_, c)| {
2380                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2381                    && c.failed.is_none()
2382                    && !c.empty
2383            })
2384            .map(|(i, _)| i)
2385            .collect();
2386        if todo.is_empty() {
2387            return self.after_implement();
2388        }
2389        self.state.status = RunStatus::Implementing;
2390
2391        let language = self.state.config.graph.language.clone();
2392        let github_language = self.state.config.graph.github_language.clone();
2393        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2394        let sessions = self.state.config.graph.sessions;
2395        let artifacts = agent::artifacts_dir(&self.state.dir());
2396        // The design-deliberation stage's blended brief, when `advise` found
2397        // one — carried into every implementer's prompt the same way
2398        // regardless of which candidate it is.
2399        let brief = self
2400            .state
2401            .advice
2402            .as_ref()
2403            .and_then(|a| a.synthesis.as_deref())
2404            .map(str::to_owned);
2405        let attachments = self.state.attachments.clone();
2406
2407        let mut jobs = Vec::new();
2408        for &i in &todo {
2409            let (index, label, worktree) = {
2410                let c = &self.state.candidates[i];
2411                (c.index, c.label, c.worktree.clone())
2412            };
2413            let spec = self.roles.implementers[index].clone();
2414            let seat_key = format!("impl-{label}");
2415            let seat = self.seat(&seat_key, &spec.id);
2416            let instruction = seeded_instruction(&self.state);
2417            jobs.push(SeatJob {
2418                spec,
2419                seat,
2420                prompt: prompt::implement_in(
2421                    &instruction,
2422                    &worktree.to_string_lossy(),
2423                    &language,
2424                    &github_language,
2425                    brief.as_deref(),
2426                    &attachments,
2427                ),
2428                cwd: worktree,
2429                timeout,
2430                allow_write: true,
2431                sessions,
2432                artifacts: artifacts.clone(),
2433                stem: format!("impl-{label}"),
2434                handover: None,
2435            });
2436        }
2437
2438        self.state.event(
2439            "implement",
2440            format!("{} candidates in parallel", jobs.len()),
2441        );
2442        // Kept so a seat whose CLI hung up can be asked again from the same
2443        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2444        // can update a seat's own entry once a fallback agent takes it over —
2445        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2446        // whichever agent actually answered, not the one that quota'd out.
2447        let mut sent = jobs.clone();
2448        let cache = self.state.config.cache_dir();
2449        let ctx = WaveCtx {
2450            carry_seats: false,
2451            pauses: Some((&self.pause, &self.interrupt)),
2452            run: &run_id,
2453            node: "implement",
2454            prompts: &prompts,
2455            cache: cache.as_deref(),
2456            round: None,
2457        };
2458        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2459        if !wave_blocked(&results) {
2460            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2461                .await;
2462            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2463                .await?;
2464            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2465                .await;
2466        }
2467        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2468
2469        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2470            && results
2471                .iter()
2472                .any(|(_, _, out)| FailClass::of(out).is_some());
2473        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2474            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2475                // Executed turns interrupted by disk pressure already carry
2476                // a preservation barrier from wave. Admission-only waits do
2477                // not. Keep the interrupted seat's session for recovery too.
2478                if self
2479                    .state
2480                    .candidate_preservation
2481                    .contains_key(&self.state.candidates[i].label)
2482                {
2483                    self.state.seats.insert(seat.key.clone(), seat);
2484                }
2485                continue;
2486            }
2487            let seat_key = seat.key.clone();
2488            // A quota fallback (`resume_seat_handovers`) may have handed this
2489            // seat to a different agent than the one `prep` recorded on the
2490            // candidate; the stats tables and any later fixer-defaults-to-
2491            // winner's-author lookup must credit whoever actually answered —
2492            // unless every fallback also quota'd out, in which case nobody
2493            // actually answered and crediting the last agent tried would
2494            // erase every earlier agent's own quota loss from the stats
2495            // tables instead of just this one seat's.
2496            let agent = seat.agent.clone();
2497            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2498            self.state.seats.insert(seat.key.clone(), seat);
2499            let label = self.state.candidates[i].label;
2500            self.state.interrupted_implementations.remove(&label);
2501
2502            let (summary, duration, failed, verified_claim) = match out {
2503                AgentOutcome::Ok(o) => {
2504                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2505                    let failed = (!o.usable()).then(|| {
2506                        if o.timed_out {
2507                            "agent timed out".to_owned()
2508                        } else {
2509                            format!(
2510                                "agent exited with {}",
2511                                crate::notices::exit_status(o.exit_code)
2512                            )
2513                        }
2514                    });
2515                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2516                    (text, o.duration_ms, failed, verified_claim)
2517                }
2518                // Left un-resumed by `resume_undelivered` (a dirty tree
2519                // already rescues the work, or there was no session left to
2520                // resume into) — reported like the ordinary failure it is,
2521                // never as if `o.text` (the CLI's raw error JSON) were an
2522                // answer.
2523                AgentOutcome::Dropped(o) => {
2524                    let why = o
2525                        .dropped
2526                        .as_ref()
2527                        .map(|d| d.why.as_str())
2528                        .unwrap_or("the CLI ended the stream without delivering its answer");
2529                    (
2530                        String::new(),
2531                        o.duration_ms,
2532                        Some(format!("the CLI dropped the stream ({why})")),
2533                        None,
2534                    )
2535                }
2536                AgentOutcome::Quota(o) => {
2537                    self.state.quota.push(QuotaLoss {
2538                        seat: seat_key,
2539                        node: "implement".to_owned(),
2540                        at: Timestamp::now(),
2541                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2542                    });
2543                    (
2544                        String::new(),
2545                        o.duration_ms,
2546                        Some("rate limited (quota); produced no change".to_owned()),
2547                        None,
2548                    )
2549                }
2550                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2551                    (String::new(), 0, Some(e), None)
2552                }
2553            };
2554
2555            // Save every completed turn before any fallible Git operation. A failed
2556            // rescue must not erase the reply or prevent collection of other seats.
2557            let c = &mut self.state.candidates[i];
2558            if !exhausted_the_fallback_chain {
2559                c.agent = agent;
2560            }
2561            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2562            c.duration_ms = duration;
2563            c.failed = failed;
2564            c.verified_noop = verified_claim;
2565            self.state
2566                .candidate_preservation
2567                .insert(label, "awaiting persistence".to_owned());
2568        }
2569        self.state.save()?;
2570        self.persist_candidates().await?;
2571
2572        if resource_blocked {
2573            return self.stop_for_resource();
2574        }
2575        if selected_failed {
2576            self.state.status = RunStatus::Blocked;
2577            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2578            self.state.save()?;
2579            bail!(
2580                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2581            );
2582        }
2583        self.after_implement()
2584    }
2585
2586    /// Retry only persistence after a completed implementation turn.
2587    async fn persist_candidates(&mut self) -> Result<()> {
2588        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2589        for label in labels {
2590            let i = self
2591                .state
2592                .candidates
2593                .iter()
2594                .position(|c| c.label == label)
2595                .context("pending candidate is missing")?;
2596            let c = self.state.candidates[i].clone();
2597            let interrupted = self.state.interrupted_implementations.contains(&label);
2598            let result = async {
2599                let wait = crate::repo_lock::WaitPolicy::new(
2600                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2601                    || false,
2602                );
2603                let _coordination =
2604                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2605                // Best effort evidence requires no index writes. A successful snapshot
2606                // never authorizes removal of a tree whose rescue failed.
2607                if let Err(e) = git::snapshot_uncommitted(
2608                    &c.worktree,
2609                    &self
2610                        .state
2611                        .dir()
2612                        .join("artifacts")
2613                        .join(format!("candidate-{label}-recovery")),
2614                )
2615                .await
2616                {
2617                    self.state.event(
2618                        "preserve",
2619                        format!("candidate {label} snapshot incomplete: {e:#}"),
2620                    );
2621                }
2622                let rescued = git::rescue_commit(
2623                    &c.worktree,
2624                    &format!("magi: candidate {label} (uncommitted work)"),
2625                )
2626                .await?;
2627                self.state.note_withheld("implement", &rescued.withheld);
2628                let base = &self.state.base_commit;
2629                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2630                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2631                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2632                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2633                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2634                if c.empty && commits > 0 && !patch.trim().is_empty()
2635                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2636                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2637                        || self.state.tally.is_some())
2638                {
2639                    // Keep the old classification and pending marker so every
2640                    // retry stops too. Retain decisions as recovery evidence;
2641                    // only a supported recovery operation may replace them.
2642                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2643                }
2644                let c = &mut self.state.candidates[i];
2645                c.commits = commits;
2646                c.stat = stat;
2647                c.files = files;
2648                let no_changes = commits == 0 || patch.trim().is_empty();
2649                c.empty = no_changes && !interrupted;
2650                if interrupted && no_changes {
2651                    c.failed = None;
2652                    c.verified_noop = None;
2653                }
2654                if !c.empty {
2655                    c.failed = None;
2656                    c.verified_noop = None;
2657                }
2658                self.state.event(
2659                    "preserve",
2660                    format!("candidate {label}: {files} files, {commits} commits"),
2661                );
2662                Ok::<_, anyhow::Error>(())
2663            }
2664            .await;
2665            match result {
2666                Ok(()) => {
2667                    self.state.candidate_preservation.remove(&label);
2668                    if !interrupted || self.state.candidates[i].files > 0 {
2669                        self.state.interrupted_implementations.remove(&label);
2670                    }
2671                }
2672                Err(e) => {
2673                    let reason = format!("candidate {label} preservation failed: {e:#}");
2674                    self.state.event("preserve", &reason);
2675                    self.state.candidate_preservation.insert(label, reason);
2676                }
2677            }
2678            if self.state.candidate_preservation.is_empty()
2679                && self.state.status == RunStatus::Blocked
2680                && self.state.candidates.iter().all(|c| c.failed.is_none())
2681            {
2682                // Persist the recovered phase with the cleared pending marker;
2683                // a crash between saves must not turn this into a selected-seat
2684                // failure hold on the next resume.
2685                self.state.status = RunStatus::Implementing;
2686                self.state.event(
2687                    "preserve",
2688                    "candidate persistence recovered; implementation can continue",
2689                );
2690            }
2691            self.state.save()?;
2692        }
2693        if !self.state.candidate_preservation.is_empty() {
2694            self.state.status = RunStatus::Blocked;
2695            self.state.save()?;
2696            bail!(
2697                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2698            );
2699        }
2700        Ok(())
2701    }
2702
2703    /// Ask again, once, for work a CLI did and then failed to hand over.
2704    ///
2705    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2706    /// status with an empty response and a usage report showing output tokens,
2707    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2708    /// seven minutes and 14,267 output tokens that arrived as an empty
2709    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2710    ///
2711    /// Two conditions, and both matter:
2712    ///
2713    /// - **Only when the tree is untouched.** Often the agent has already
2714    ///   written its files and only the closing message was lost; the rescue
2715    ///   commit below picks that up and there is nothing to ask for. Re-asking
2716    ///   then would pay for a second implementation of work already on disk.
2717    /// - **Once.** A CLI that drops one stream can drop the next, and this
2718    ///   node is the most expensive in the graph.
2719    ///
2720    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2721    /// dropped reply still carried its `conversation_id`, so the seat is asked
2722    /// to finish what it was doing rather than sent the whole task again. It
2723    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2724    /// node's - for the same reason a re-ranked judge does: restating finished
2725    /// work is not the work.
2726    ///
2727    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2728    /// same way until it resets, while an abandoned conversation is still
2729    /// there to be picked up.
2730    async fn resume_undelivered(
2731        &mut self,
2732        results: &mut [(usize, SeatState, AgentOutcome)],
2733        sent: &[SeatJob],
2734        prompts: &Prompts,
2735        run_id: &str,
2736    ) {
2737        for (wi, seat, out) in results.iter_mut() {
2738            let Some(dropped) = (match &*out {
2739                AgentOutcome::Dropped(o) => o.dropped.clone(),
2740                _ => None,
2741            }) else {
2742                continue;
2743            };
2744            let Some(job) = sent.get(*wi) else { continue };
2745            // Already on disk? Then only the closing message was lost.
2746            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2747                self.state.event(
2748                    "implement",
2749                    format!(
2750                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2751                         work is in the tree",
2752                        seat.key, dropped.output_tokens, dropped.why
2753                    ),
2754                );
2755                continue;
2756            }
2757            // The re-ask only makes sense as a resume: `resume_after_drop`
2758            // says nothing about the task, trusting the seat to still hold it.
2759            // Without a session to resume — sessions disabled, or this CLI's
2760            // drop shape happened not to carry a session id — that prompt
2761            // would open a brand-new conversation with no context at all,
2762            // which is worse than leaving this as the ordinary failure it
2763            // already is.
2764            if !has_context(&job.spec, seat, job.sessions) {
2765                self.state.event(
2766                    "implement",
2767                    format!(
2768                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2769                         is no session left to resume",
2770                        seat.key, dropped.output_tokens, dropped.why
2771                    ),
2772                );
2773                continue;
2774            }
2775            self.state.event(
2776                "implement",
2777                format!(
2778                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2779                     conversation",
2780                    seat.key, dropped.output_tokens, dropped.why
2781                ),
2782            );
2783            let mut retry = job.clone();
2784            retry.seat = seat.clone();
2785            retry.prompt = prompt::resume_after_drop(&dropped.why);
2786            retry.timeout = retry_budget(job.timeout, true);
2787            retry.stem = format!("{}-resume", job.stem);
2788            let cache = self.state.config.cache_dir();
2789            let ctx = WaveCtx {
2790                carry_seats: false,
2791                pauses: Some((&self.pause, &self.interrupt)),
2792                run: run_id,
2793                node: "implement",
2794                prompts,
2795                cache: cache.as_deref(),
2796                round: None,
2797            };
2798            let (resumed_seat, resumed) =
2799                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2800            *seat = resumed_seat;
2801            *out = resumed;
2802        }
2803    }
2804
2805    /// Fall an implement seat through to the next untried agent in the
2806    /// implementer roster when it lost to quota — or, since the handover was
2807    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2808    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2809    /// quota path itself being unchanged — instead of leaving the
2810    /// seat's loss final the moment one agent's account runs dry.
2811    ///
2812    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2813    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2814    /// truncates to the single slot rotation picked, so a solo task whose one
2815    /// implementer hits quota mid-run used to have nothing else to try. This
2816    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2817    /// unrotated roster — which is the only place the *other* candidates in
2818    /// the machine's roster still exist once `implementers` has been cut down
2819    /// to size.
2820    ///
2821    /// Walks forward from just past the seat's own original position in the
2822    /// roster, never wrapping back to the front: a later candidate slot (say
2823    /// `beta`, the roster's second entry) must fall through to the *next*
2824    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2825    /// almost certainly a different candidate's own agent already — and once
2826    /// the roster's tail is exhausted there is nothing left to fall through
2827    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2828    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2829    /// let this retry that id forever. The loop keeps falling through until
2830    /// an attempt lands something other than `Quota` or the roster's tail
2831    /// runs out of untried ids, at which point the seat is left exactly as
2832    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2833    /// `QuotaLoss` recorded, the candidate failed/empty.
2834    ///
2835    /// `sent` is taken mutably and updated with the fallback agent's spec:
2836    /// `resume_unconfirmed_commands`, which runs after this and also reads
2837    /// `sent`, must see whichever agent actually ended up answering the seat
2838    /// — reading the stale, original spec there would check session
2839    /// eligibility against the wrong CLI and could hand a fallback agent's
2840    /// session id to the agent that just lost the seat to quota.
2841    ///
2842    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2843    /// — `self.seat` only reuses state when the agent id is unchanged, so
2844    /// handing it a different id already gets this for free. Reusing the old
2845    /// seat would resume a different CLI's session as if it were a
2846    /// continuation of this one.
2847    ///
2848    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2849    /// a quota loss cuts an agent off mid-turn, so anything already in the
2850    /// tree is unfinished work, not a completed candidate a re-ask would pay
2851    /// for twice. A dirty tree is rescued into a commit first (the same
2852    /// neutral-identity rescue `implement`'s own outcome loop gives every
2853    /// candidate) so the next agent starts clean.
2854    ///
2855    /// The new agent gets the implementer's full prompt and full
2856    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2857    /// it has no session and no context, and is implementing the task from
2858    /// nothing, unlike a resumed drop which is only restating work already
2859    /// done.
2860    ///
2861    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2862    /// `implement` event, never into `self.state.quota` — that is what
2863    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2864    /// go unspent, and a seat that ultimately recovered on its second or
2865    /// third agent is not the stalled panel that check exists to catch. Only
2866    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2867    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2868    /// outcome loop already has — this helper never pushes to it itself.
2869    async fn resume_seat_handovers(
2870        &mut self,
2871        results: &mut [(usize, SeatState, AgentOutcome)],
2872        sent: &mut [SeatJob],
2873        prompts: &Prompts,
2874        run_id: &str,
2875    ) -> Result<()> {
2876        let instruction = seeded_instruction(&self.state);
2877        let language = self.state.config.graph.language.clone();
2878        let github_language = self.state.config.graph.github_language.clone();
2879        let brief = self
2880            .state
2881            .advice
2882            .as_ref()
2883            .and_then(|a| a.synthesis.as_deref())
2884            .map(str::to_owned);
2885        let attachments = self.state.attachments.clone();
2886        for (wi, seat, out) in results.iter_mut() {
2887            // Who holds the other candidate seats of this wave right now
2888            // (earlier handovers already written back to `sent`).
2889            let others: BTreeSet<String> = sent
2890                .iter()
2891                .enumerate()
2892                .filter(|(j, _)| j != wi)
2893                .map(|(_, j)| j.spec.id.clone())
2894                .collect();
2895            let Some(job) = sent.get_mut(*wi) else {
2896                continue;
2897            };
2898            // Where the seat's own original agent sits in the roster — the
2899            // fallback walk starts just past here, never at the front, so a
2900            // later candidate slot's quota loss does not fall back onto an
2901            // earlier slot's own agent.
2902            let start = self
2903                .roles
2904                .implementer_roster
2905                .iter()
2906                .position(|s| s.id == job.spec.id)
2907                .unwrap_or(0);
2908            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2909            let mut fallback_attempt = 0usize;
2910            let mut prev: Option<FailClass> = None;
2911            while let Some(cur) = FailClass::of(&*out) {
2912                if !should_hand_over(prev.as_ref(), &cur) {
2913                    break;
2914                }
2915                let Some(next) =
2916                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2917                        .cloned()
2918                else {
2919                    break;
2920                };
2921                tried.insert(next.id.clone());
2922                fallback_attempt += 1;
2923
2924                let coordination = self.coordinate(&job.cwd).await?;
2925                if let Ok(r) = git::rescue_commit(
2926                    &job.cwd,
2927                    &format!(
2928                        "magi: candidate {} (uncommitted work before {} fallback)",
2929                        seat.key,
2930                        if cur == FailClass::Quota {
2931                            "quota"
2932                        } else {
2933                            "handover"
2934                        }
2935                    ),
2936                )
2937                .await
2938                {
2939                    self.state.note_withheld("implement", &r.withheld);
2940                }
2941                drop(coordination);
2942
2943                record_handover(
2944                    &mut self.state,
2945                    "implement",
2946                    &seat.key,
2947                    &seat.agent,
2948                    &next.id,
2949                    &cur,
2950                    &fail_reason(&*out),
2951                );
2952                prev = Some(cur.clone());
2953
2954                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2955                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2956                // Kept in sync on `sent` itself, not just the local retry: a
2957                // later helper (`resume_unconfirmed_commands`) reads `sent`
2958                // after this one returns and must see whichever agent is now
2959                // occupying the seat, not the one that just quota'd out —
2960                // otherwise it would judge session/continuation eligibility
2961                // by the wrong CLI and could resend a fallback's session id
2962                // to the agent that lost it the seat in the first place.
2963                job.spec = next.clone();
2964                let mut retry = job.clone();
2965                retry.seat = new_seat;
2966                retry.prompt = prompt::implement_in(
2967                    &instruction,
2968                    &job.cwd.to_string_lossy(),
2969                    &language,
2970                    &github_language,
2971                    brief.as_deref(),
2972                    &attachments,
2973                );
2974                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2975                let cache = self.state.config.cache_dir();
2976                let ctx = WaveCtx {
2977                    carry_seats: false,
2978                    pauses: Some((&self.pause, &self.interrupt)),
2979                    run: run_id,
2980                    node: "implement",
2981                    prompts,
2982                    cache: cache.as_deref(),
2983                    round: None,
2984                };
2985                let (fallback_seat, fallback_out) = run_one(
2986                    retry,
2987                    Arc::clone(&self.sem),
2988                    &ctx,
2989                    &mut self.state,
2990                    fallback_attempt,
2991                )
2992                .await;
2993                *seat = fallback_seat;
2994                *out = fallback_out;
2995            }
2996        }
2997        Ok(())
2998    }
2999
3000    /// Ask an implement seat's own CLI to confirm what it started, once, when
3001    /// its reply reported a command whose completion status it never
3002    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3003    /// that does and does not mean.
3004    ///
3005    /// The completion contract this task asks for, extended to `implement`
3006    /// with the same signal `continue_fix_report` reads for the fixer,
3007    /// rather than a keyword search over the reply or a hard requirement on
3008    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3009    /// a candidate's CLI turn ended cleanly while a test run it had started
3010    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3011    /// named in it at all is untouched by this: `commands` is empty, so
3012    /// there is nothing to be unconfirmed.
3013    ///
3014    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3015    /// this is not about recovering edits that might already be on disk, it
3016    /// is about a result the seat itself never vouched for, which resuming
3017    /// asks for regardless of what the tree already holds. Bounded to one
3018    /// attempt for the same reason `resume_undelivered` is — this is the
3019    /// most expensive node in the graph — and a seat that still cannot
3020    /// confirm on that attempt is left as whatever its (possibly still
3021    /// unconfirmed) reply says; this does not invent a new "failed" reason
3022    /// for a candidate that otherwise produced a real, committed change.
3023    async fn resume_unconfirmed_commands(
3024        &mut self,
3025        results: &mut [(usize, SeatState, AgentOutcome)],
3026        sent: &[SeatJob],
3027        prompts: &Prompts,
3028        run_id: &str,
3029    ) {
3030        for (wi, seat, out) in results.iter_mut() {
3031            let AgentOutcome::Ok(o) = &*out else {
3032                continue;
3033            };
3034            if !has_unconfirmed_command(&o.commands) {
3035                continue;
3036            }
3037            let Some(job) = sent.get(*wi) else { continue };
3038            if !has_context(&job.spec, seat, job.sessions) {
3039                self.state.event(
3040                    "implement",
3041                    format!(
3042                        "{}: the reply named a command whose own CLI never confirmed the exit \
3043                         status of, but there is no session left to resume",
3044                        seat.key
3045                    ),
3046                );
3047                continue;
3048            }
3049            self.state.event(
3050                "implement",
3051                format!(
3052                    "{}: the reply named a command whose own CLI never confirmed the exit \
3053                     status of; resuming the conversation",
3054                    seat.key
3055                ),
3056            );
3057            let mut retry = job.clone();
3058            retry.seat = seat.clone();
3059            retry.prompt = prompt::resume_incomplete(
3060                "a command in your last reply had no confirmed exit status",
3061            );
3062            retry.timeout = retry_budget(job.timeout, true);
3063            retry.stem = format!("{}-confirm", job.stem);
3064            let cache = self.state.config.cache_dir();
3065            let ctx = WaveCtx {
3066                carry_seats: false,
3067                pauses: Some((&self.pause, &self.interrupt)),
3068                run: run_id,
3069                node: "implement",
3070                prompts,
3071                cache: cache.as_deref(),
3072                round: None,
3073            };
3074            let (resumed_seat, resumed) =
3075                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3076            *seat = resumed_seat;
3077            *out = resumed;
3078        }
3079    }
3080
3081    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3082    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3083    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3084    /// that motivated this.
3085    ///
3086    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3087    /// own nudge loop already covers for judge/review/vote seats, and not a
3088    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3089    /// implement seats: here the CLI turn genuinely finished while the node's
3090    /// own work — the fixer's account of what it did — had not. Gated purely
3091    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3092    /// reply, never on any wording in it, so a fixer whose valid, first-try
3093    /// `FixReport` happens to mention having waited on a background test is
3094    /// never resumed — the `Ok(report)` branch at the call site returns
3095    /// before this is ever invoked.
3096    ///
3097    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3098    /// attempt ([`retry_budget`]), nothing attempted once the session is
3099    /// gone, and a quota hit ends the loop immediately rather than retrying a
3100    /// rate limit that fails the same way again.
3101    async fn continue_fix_report(
3102        &mut self,
3103        mut seat: SeatState,
3104        parse_err: String,
3105        job: &SeatJob,
3106        prompts: &Prompts,
3107        run_id: &str,
3108        round: usize,
3109    ) -> (
3110        SeatState,
3111        Option<FixReport>,
3112        Option<String>,
3113        ContinuationRecord,
3114    ) {
3115        let mut last_err = parse_err;
3116        let mut cumulative_wait_ms = 0u64;
3117        let mut attempts = 0usize;
3118        loop {
3119            if !has_context(&job.spec, &seat, job.sessions) {
3120                self.state.event(
3121                    "fix",
3122                    format!(
3123                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3124                         session left to resume into"
3125                    ),
3126                );
3127                let outcome = if attempts == 0 {
3128                    ContinuationOutcome::NoSession
3129                } else {
3130                    ContinuationOutcome::Exhausted
3131                };
3132                return (
3133                    seat,
3134                    None,
3135                    Some(format!("unparsable fix report: {last_err}")),
3136                    ContinuationRecord {
3137                        attempts,
3138                        cumulative_wait_ms,
3139                        outcome,
3140                    },
3141                );
3142            }
3143            if attempts >= MAX_FIX_CONTINUATIONS {
3144                self.state.event(
3145                    "fix",
3146                    format!(
3147                        "round {round}: fixer's reply still had no adoption report after \
3148                         {attempts} continuation(s) ({last_err}); giving up"
3149                    ),
3150                );
3151                return (
3152                    seat,
3153                    None,
3154                    Some(format!(
3155                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3156                    )),
3157                    ContinuationRecord {
3158                        attempts,
3159                        cumulative_wait_ms,
3160                        outcome: ContinuationOutcome::Exhausted,
3161                    },
3162                );
3163            }
3164            attempts += 1;
3165            self.state.event(
3166                "fix",
3167                format!(
3168                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3169                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3170                ),
3171            );
3172            let mut retry = job.clone();
3173            retry.seat = seat.clone();
3174            retry.prompt = prompt::resume_incomplete(&last_err);
3175            retry.timeout = retry_budget(job.timeout, true);
3176            retry.stem = format!("{}-continue{attempts}", job.stem);
3177            let cache = self.state.config.cache_dir();
3178            let ctx = WaveCtx {
3179                carry_seats: false,
3180                pauses: Some((&self.pause, &self.interrupt)),
3181                run: run_id,
3182                node: "fix",
3183                prompts,
3184                cache: cache.as_deref(),
3185                round: Some(round),
3186            };
3187            let (resumed_seat, resumed_out) = run_one(
3188                retry,
3189                Arc::clone(&self.sem),
3190                &ctx,
3191                &mut self.state,
3192                attempts,
3193            )
3194            .await;
3195            seat = resumed_seat;
3196            match resumed_out {
3197                AgentOutcome::Ok(o) => {
3198                    cumulative_wait_ms += o.duration_ms;
3199                    match verdict::extract_json::<FixReport>(&o.text) {
3200                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3201                            self.state.event(
3202                                "fix",
3203                                format!(
3204                                    "round {round}: fixer's adoption report recovered after \
3205                                     {attempts} continuation(s)"
3206                                ),
3207                            );
3208                            return (
3209                                seat,
3210                                Some(report),
3211                                None,
3212                                ContinuationRecord {
3213                                    attempts,
3214                                    cumulative_wait_ms,
3215                                    outcome: ContinuationOutcome::Resumed,
3216                                },
3217                            );
3218                        }
3219                        // The report parsed, but this same reply's own
3220                        // CommandEvidence — the identical record `state.jobs`
3221                        // renders — names a command whose CLI never
3222                        // confirmed an exit status. Read together, that is
3223                        // not a resolved answer: keep nudging rather than
3224                        // accept a report standing next to a command the
3225                        // seat's own CLI cannot vouch for.
3226                        Ok(_) => {
3227                            last_err = "the reply parsed, but it reported a command whose own CLI \
3228                                 never confirmed an exit status"
3229                                .to_owned();
3230                        }
3231                        Err(e) => last_err = e.to_string(),
3232                    }
3233                }
3234                AgentOutcome::Quota(o) => {
3235                    cumulative_wait_ms += o.duration_ms;
3236                    self.state.quota.push(QuotaLoss {
3237                        seat: seat.key.clone(),
3238                        node: "fix".to_owned(),
3239                        at: Timestamp::now(),
3240                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3241                    });
3242                    self.state.event(
3243                        "fix",
3244                        format!(
3245                            "round {round}: continuation rate limited (quota); not retrying now"
3246                        ),
3247                    );
3248                    return (
3249                        seat,
3250                        None,
3251                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3252                        ContinuationRecord {
3253                            attempts,
3254                            cumulative_wait_ms,
3255                            outcome: ContinuationOutcome::QuotaLost,
3256                        },
3257                    );
3258                }
3259                AgentOutcome::Dropped(o) => {
3260                    cumulative_wait_ms += o.duration_ms;
3261                    let why = o
3262                        .dropped
3263                        .as_ref()
3264                        .map(|d| d.why.as_str())
3265                        .unwrap_or("the CLI ended the stream without delivering its answer");
3266                    last_err = format!("the CLI dropped the stream ({why})");
3267                }
3268                AgentOutcome::Failed(e) => last_err = e,
3269                // The continuation was never started; the report stays lost
3270                // and the caller records that, without any handover.
3271                AgentOutcome::ResourceBlocked(e) => {
3272                    return (
3273                        seat,
3274                        None,
3275                        Some(format!("the continuation was not started: {e}")),
3276                        ContinuationRecord {
3277                            attempts: attempts.saturating_sub(1),
3278                            cumulative_wait_ms,
3279                            outcome: ContinuationOutcome::NoSession,
3280                        },
3281                    );
3282                }
3283            }
3284        }
3285    }
3286
3287    fn after_implement(&mut self) -> Result<()> {
3288        if !self.state.candidate_preservation.is_empty() {
3289            bail!("candidate preservation incomplete; judging is unsafe");
3290        }
3291        // Scan every candidate patch once the set is complete.
3292        if self.state.leaks.is_empty() {
3293            let cfg = self.state.config.blind.clone();
3294            let mut leaks = Vec::new();
3295            for c in &self.state.candidates {
3296                let Some(patch) =
3297                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3298                else {
3299                    continue;
3300                };
3301                leaks.extend(blind::scan(
3302                    &format!("candidate {} patch", c.label),
3303                    &patch,
3304                    &cfg.vendor_tokens,
3305                ));
3306            }
3307            if !leaks.is_empty() {
3308                let summary = leaks
3309                    .iter()
3310                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3311                    .collect::<Vec<_>>()
3312                    .join(", ");
3313                match cfg.on_leak {
3314                    LeakPolicy::Fail => {
3315                        self.state.status = RunStatus::Failed;
3316                        self.state
3317                            .event("blind", format!("vendor text in a patch: {summary}"));
3318                        self.state.leaks = leaks;
3319                        self.state.save()?;
3320                        self.settle_questions();
3321                        bail!(
3322                            "blind.on_leak = \"fail\" and vendor text reached a \
3323                             judged patch: {summary}"
3324                        );
3325                    }
3326                    LeakPolicy::Redact => self.state.event(
3327                        "blind",
3328                        format!("redacting vendor text for judging: {summary}"),
3329                    ),
3330                    LeakPolicy::Warn => self.state.event(
3331                        "blind",
3332                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3333                    ),
3334                }
3335                self.state.leaks = leaks;
3336            }
3337        }
3338
3339        if self.state.viable().is_empty() {
3340            if self.state.all_candidates_verified_noop() {
3341                // Every candidate agreed, with evidence the adoption guard
3342                // accepted, that nothing belongs in this worktree. That is
3343                // not the same fact as a candidate that simply failed to
3344                // write anything, and settling it as an ordinary `Failed`
3345                // (see `SCHEMA`'s doc for schema 10) is what let two of
3346                // task 391f's attempts burn a retry each re-discovering the
3347                // same already-landed fix. Terminal either way, so `judge`
3348                // must never run over an empty candidate set — unlike the
3349                // `Failed` branch below this returns `Ok`, not an error:
3350                // nothing here failed.
3351                self.state.status = RunStatus::VerifiedNoop;
3352                self.state.save()?;
3353                self.settle_questions();
3354                return Ok(());
3355            }
3356            self.state.status = RunStatus::Failed;
3357            self.state.save()?;
3358            self.settle_questions();
3359            bail!("no candidate produced a change; nothing to judge");
3360        }
3361        self.state.status = RunStatus::Judging;
3362        self.state.save()?;
3363        Ok(())
3364    }
3365
3366    // --------------------------------------------------------------- judge
3367
3368    async fn judge(&mut self) -> Result<()> {
3369        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3370        // agent files with `magi task add` name the run that paid for it. The
3371        // prompt overlay is cloned alongside it because the waves borrow it
3372        // while `self` is mutably borrowed by the node's own bookkeeping.
3373        let run_id = self.state.id.clone();
3374        let prompts = self.state.config.prompts.clone();
3375        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3376            return Ok(());
3377        }
3378        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3379        if viable.len() == 1 {
3380            // Recorded so this is a one-time event: `judgements` stays empty
3381            // either way, which without this flag is indistinguishable from
3382            // "not yet judged" on the next reentry — and status is left
3383            // untouched, so a later node's conclusion (e.g. `Blocked` after
3384            // the review budget ran out) survives a resume instead of being
3385            // clobbered back to `Judging` by this node running again.
3386            self.state.judge_skipped = true;
3387            self.state.event(
3388                "judge",
3389                format!(
3390                    "only candidate {} produced a change; judging skipped",
3391                    viable[0].label
3392                ),
3393            );
3394            self.state.save()?;
3395            return Ok(());
3396        }
3397        self.state.status = RunStatus::Judging;
3398
3399        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3400        let language = self.state.config.graph.language.clone();
3401        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3402        let sessions = self.state.config.graph.sessions;
3403        let artifacts = agent::artifacts_dir(&self.state.dir());
3404        let root = self.state.worktree_root();
3405        let base_short = short(&self.state.base_commit);
3406
3407        let mut jobs = Vec::new();
3408        let mut orders = Vec::new();
3409        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3410            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3411            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3412            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3413            let seat_key = format!("judge-{}", j + 1);
3414            let seat = self.seat(&seat_key, &spec.id);
3415            jobs.push(SeatJob {
3416                prompt: prompt::judge(
3417                    &self.state.instruction,
3418                    &views,
3419                    self.roles.judges.len(),
3420                    &base_short,
3421                    &language,
3422                ),
3423                spec,
3424                seat,
3425                cwd: root.join(format!("judge-{}", j + 1)),
3426                timeout,
3427                allow_write: false,
3428                sessions,
3429                artifacts: artifacts.clone(),
3430                stem: format!("judge-{}", j + 1),
3431                handover: None,
3432            });
3433        }
3434
3435        self.state.event(
3436            "judge",
3437            format!(
3438                "{} judges ranking {} candidates blind",
3439                jobs.len(),
3440                viable.len()
3441            ),
3442        );
3443        let labels_for_check = labels.clone();
3444        let mut quota_losses = Vec::new();
3445        let cache = self.state.config.cache_dir();
3446        let ctx = WaveCtx {
3447            carry_seats: false,
3448            pauses: Some((&self.pause, &self.interrupt)),
3449            run: &run_id,
3450            node: "judge",
3451            prompts: &prompts,
3452            cache: cache.as_deref(),
3453            round: None,
3454        };
3455        let results = ask_json_wave::<Ranking>(
3456            jobs,
3457            Arc::clone(&self.sem),
3458            self.state.config.graph.retries,
3459            &self.roles.judge_roster,
3460            &ctx,
3461            &mut quota_losses,
3462            &mut self.state,
3463            &move |r: &Ranking| r.validate(&labels_for_check),
3464        )
3465        .await;
3466        if self.state.resource_block.is_some() {
3467            return self.stop_for_resource();
3468        }
3469        self.state.quota.extend(quota_losses);
3470
3471        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3472            let agent_id = seat.agent.clone();
3473            self.state.seats.insert(seat.key.clone(), seat);
3474            let mut record = Judgement {
3475                judge: j + 1,
3476                seat: format!("judge-{}", j + 1),
3477                agent: agent_id,
3478                ranking: Vec::new(),
3479                reasons: BTreeMap::new(),
3480                confidence: None,
3481                strengths: BTreeMap::new(),
3482                order: orders[j].clone(),
3483                failed: None,
3484                duration_ms: 0,
3485            };
3486            match res {
3487                Ok((ranking, out)) => {
3488                    record.ranking = ranking.normalized();
3489                    record.reasons = ranking.reasons;
3490                    record.strengths = ranking.strengths;
3491                    record.confidence = ranking.confidence;
3492                    record.duration_ms = out.duration_ms;
3493                    self.state.event(
3494                        "judge",
3495                        format!(
3496                            "judge {} ranked {}",
3497                            j + 1,
3498                            record.ranking.iter().collect::<String>()
3499                        ),
3500                    );
3501                }
3502                Err(e) => {
3503                    record.failed = Some(e.to_string());
3504                    self.state
3505                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3506                }
3507            }
3508            self.state.judgements.push(record);
3509            self.state.save()?;
3510        }
3511        Ok(())
3512    }
3513
3514    // ---------------------------------------------------------- deliberate
3515
3516    async fn deliberate(&mut self) -> Result<()> {
3517        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3518        // agent files with `magi task add` name the run that paid for it. The
3519        // prompt overlay is cloned alongside it because the waves borrow it
3520        // while `self` is mutably borrowed by the node's own bookkeeping.
3521        let run_id = self.state.id.clone();
3522        let prompts = self.state.config.prompts.clone();
3523        if !self.state.deliberation.is_empty() {
3524            return Ok(());
3525        }
3526        let tops: Vec<char> = self
3527            .state
3528            .judgements
3529            .iter()
3530            .filter_map(|j| j.ranking.first().copied())
3531            .collect();
3532        let rounds = self.state.config.graph.deliberate_rounds;
3533        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3534            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3535                self.state.event(
3536                    "deliberate",
3537                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3538                );
3539            }
3540            self.state.status = RunStatus::Voting;
3541            self.state.save()?;
3542            return Ok(());
3543        }
3544
3545        self.state.status = RunStatus::Deliberating;
3546        self.state.event(
3547            "deliberate",
3548            format!(
3549                "split: first choices were {} — opening {rounds} round(s)",
3550                tops.iter().collect::<String>()
3551            ),
3552        );
3553
3554        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3555        let language = self.state.config.graph.language.clone();
3556        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3557        let sessions = self.state.config.graph.sessions;
3558        let artifacts = agent::artifacts_dir(&self.state.dir());
3559        let root = self.state.worktree_root();
3560        let base_short = short(&self.state.base_commit);
3561
3562        // Judges argue in sequence so that a turn can answer the one before it;
3563        // that is the difference between deliberation and three parallel
3564        // monologues.
3565        for round in 1..=rounds {
3566            let mut turns: Vec<DeliberationTurn> = Vec::new();
3567            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3568                if self.state.judgements[j].failed.is_some() {
3569                    continue;
3570                }
3571                let seat_key = format!("judge-{}", j + 1);
3572                let spec = self.occupant(&seat_key, spec);
3573                let mut seat = self.seat(&seat_key, &spec.id);
3574                let transcript = self.transcript(&turns, j);
3575                let build = |context: Option<&str>| {
3576                    prompt::deliberate(
3577                        &self.state.instruction,
3578                        context,
3579                        &transcript,
3580                        round,
3581                        rounds,
3582                        &language,
3583                    )
3584                };
3585                let block = self.candidate_block(&viable, &base_short);
3586                let full = build(Some(&block));
3587                let text = if has_context(&spec, &seat, sessions) {
3588                    build(None)
3589                } else {
3590                    full.clone()
3591                };
3592                let job = SeatJob {
3593                    spec,
3594                    seat: seat.clone(),
3595                    prompt: text,
3596                    cwd: root.join(format!("judge-{}", j + 1)),
3597                    timeout,
3598                    allow_write: false,
3599                    sessions,
3600                    artifacts: artifacts.clone(),
3601                    stem: format!("delib-{round}-judge-{}", j + 1),
3602                    handover: Some(full),
3603                };
3604                let cache = self.state.config.cache_dir();
3605                let ctx = WaveCtx {
3606                    carry_seats: false,
3607                    pauses: Some((&self.pause, &self.interrupt)),
3608                    run: &run_id,
3609                    node: "deliberate",
3610                    prompts: &prompts,
3611                    cache: cache.as_deref(),
3612                    round: None,
3613                };
3614                // A turn is never nudged (`retries` 0): a failed seat is
3615                // handed to the next roster agent, which gets the full
3616                // context. An empty answer is a turn, not a failure.
3617                let mut losses = Vec::new();
3618                let mut results = ask_wave_with::<String>(
3619                    vec![job],
3620                    Arc::clone(&self.sem),
3621                    0,
3622                    &self.roles.judge_roster,
3623                    &ctx,
3624                    &mut losses,
3625                    &mut self.state,
3626                    &|text: &str| {
3627                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3628                    },
3629                )
3630                .await;
3631                self.state.quota.extend(losses);
3632                let (updated, res, _) = results.pop().expect("one job in, one result out");
3633                seat = updated;
3634                let agent_id = seat.agent.clone();
3635                self.state.seats.insert(seat.key.clone(), seat);
3636                let body = match res {
3637                    Ok((body, _)) => body,
3638                    // Skip the seat; a CLI's raw error JSON is never read as
3639                    // this judge's position.
3640                    Err(e) => {
3641                        self.state
3642                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3643                        continue;
3644                    }
3645                };
3646                let tentative = verdict::extract_json::<Position>(&body)
3647                    .ok()
3648                    .and_then(|p| p.tentative)
3649                    .and_then(|s| s.trim().chars().next())
3650                    .map(|c| c.to_ascii_uppercase());
3651                self.state.event(
3652                    "deliberate",
3653                    format!(
3654                        "round {round}: judge {} now favours {}",
3655                        j + 1,
3656                        tentative.map_or("—".to_owned(), |c| c.to_string())
3657                    ),
3658                );
3659                turns.push(DeliberationTurn {
3660                    judge: j + 1,
3661                    agent: agent_id,
3662                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3663                    tentative,
3664                });
3665            }
3666            self.state
3667                .deliberation
3668                .push(DeliberationRound { round, turns });
3669            self.state.save()?;
3670        }
3671
3672        self.state.status = RunStatus::Voting;
3673        self.state.save()?;
3674        Ok(())
3675    }
3676
3677    // ---------------------------------------------------------------- vote
3678
3679    async fn vote(&mut self) -> Result<()> {
3680        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3681        // agent files with `magi task add` name the run that paid for it. The
3682        // prompt overlay is cloned alongside it because the waves borrow it
3683        // while `self` is mutably borrowed by the node's own bookkeeping.
3684        let run_id = self.state.id.clone();
3685        let prompts = self.state.config.prompts.clone();
3686        if !self.state.votes.is_empty() {
3687            return Ok(());
3688        }
3689        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3690        if viable.len() == 1 {
3691            return Ok(());
3692        }
3693        self.state.status = RunStatus::Voting;
3694
3695        let language = self.state.config.graph.language.clone();
3696        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3697        let sessions = self.state.config.graph.sessions;
3698        let artifacts = agent::artifacts_dir(&self.state.dir());
3699        let root = self.state.worktree_root();
3700        let base_short = short(&self.state.base_commit);
3701        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3702
3703        let mut jobs = Vec::new();
3704        let mut seats_at = Vec::new();
3705        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3706            if self
3707                .state
3708                .judgements
3709                .get(j)
3710                .is_some_and(|r| r.failed.is_some())
3711            {
3712                continue;
3713            }
3714            let seat_key = format!("judge-{}", j + 1);
3715            let spec = self.occupant(&seat_key, spec);
3716            let seat = self.seat(&seat_key, &spec.id);
3717            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3718            let text = if has_context(&spec, &seat, sessions) {
3719                prompt::final_vote(&viable, &language)
3720            } else {
3721                full.clone()
3722            };
3723            jobs.push(SeatJob {
3724                spec,
3725                seat,
3726                prompt: text,
3727                cwd: root.join(format!("judge-{}", j + 1)),
3728                timeout,
3729                allow_write: false,
3730                sessions,
3731                artifacts: artifacts.clone(),
3732                stem: format!("vote-judge-{}", j + 1),
3733                handover: Some(full),
3734            });
3735            seats_at.push(j);
3736        }
3737
3738        self.state.event(
3739            "vote",
3740            format!(
3741                "collecting {} final votes one by one, privately",
3742                jobs.len()
3743            ),
3744        );
3745        let allowed = viable.clone();
3746        let mut quota_losses = Vec::new();
3747        let cache = self.state.config.cache_dir();
3748        let ctx = WaveCtx {
3749            carry_seats: false,
3750            pauses: Some((&self.pause, &self.interrupt)),
3751            run: &run_id,
3752            node: "vote",
3753            prompts: &prompts,
3754            cache: cache.as_deref(),
3755            round: None,
3756        };
3757        let results = ask_json_wave::<FinalVote>(
3758            jobs,
3759            Arc::clone(&self.sem),
3760            self.state.config.graph.retries,
3761            &self.roles.judge_roster,
3762            &ctx,
3763            &mut quota_losses,
3764            &mut self.state,
3765            &move |v: &FinalVote| match v.label() {
3766                Some(c) if allowed.contains(&c) => Ok(()),
3767                other => bail!("vote {other:?} is not one of {allowed:?}"),
3768            },
3769        )
3770        .await;
3771        if self.state.resource_block.is_some() {
3772            return self.stop_for_resource();
3773        }
3774        self.state.quota.extend(quota_losses);
3775
3776        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3777            let agent_id = seat.agent.clone();
3778            self.state.seats.insert(seat.key.clone(), seat);
3779            let initial = self
3780                .state
3781                .judgements
3782                .get(j)
3783                .and_then(|r| r.ranking.first().copied());
3784            let mut record = VoteRecord {
3785                judge: j + 1,
3786                agent: agent_id,
3787                vote: None,
3788                reason: String::new(),
3789                changed: false,
3790            };
3791            match res {
3792                Ok((v, _)) => {
3793                    record.vote = v.label();
3794                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3795                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3796                    self.state.event(
3797                        "vote",
3798                        format!(
3799                            "judge {} voted {}{}",
3800                            j + 1,
3801                            record.vote.unwrap_or('?'),
3802                            if record.changed { " (changed)" } else { "" }
3803                        ),
3804                    );
3805                }
3806                Err(e) => {
3807                    self.state
3808                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3809                }
3810            }
3811            self.state.votes.push(record);
3812            self.state.save()?;
3813        }
3814        Ok(())
3815    }
3816
3817    // --------------------------------------------------------------- tally
3818
3819    fn tally(&mut self) -> Result<()> {
3820        if self.state.tally.is_some() {
3821            return Ok(());
3822        }
3823        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3824        let tops: Vec<char> = self
3825            .state
3826            .judgements
3827            .iter()
3828            .filter_map(|j| j.ranking.first().copied())
3829            .collect();
3830        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3831
3832        // A judge whose private vote failed still counted once, in the initial
3833        // ranking; using it beats discarding a whole seat.
3834        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3835        let mut cast: Vec<char> = Vec::new();
3836        for (i, j) in self.state.judgements.iter().enumerate() {
3837            let vote = self
3838                .state
3839                .votes
3840                .iter()
3841                .find(|v| v.judge == i + 1)
3842                .and_then(|v| v.vote)
3843                .or_else(|| j.ranking.first().copied());
3844            if let Some(v) = vote {
3845                *first_choice.entry(v).or_insert(0) += 1;
3846                cast.push(v);
3847            }
3848        }
3849
3850        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3851        for j in &self.state.judgements {
3852            let n = j.ranking.len();
3853            for (pos, label) in j.ranking.iter().enumerate() {
3854                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3855            }
3856        }
3857
3858        let best = first_choice.values().copied().max().unwrap_or(0);
3859        let mut leaders: Vec<char> = first_choice
3860            .iter()
3861            .filter(|(_, v)| **v == best)
3862            .map(|(k, _)| *k)
3863            .collect();
3864        let mut tie_break = None;
3865        if leaders.len() > 1 {
3866            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3867            let borda_leaders: Vec<char> = leaders
3868                .iter()
3869                .copied()
3870                .filter(|l| borda[l] == top_borda)
3871                .collect();
3872            tie_break = Some(if borda_leaders.len() == 1 {
3873                format!(
3874                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3875                    leaders.len()
3876                )
3877            } else {
3878                format!(
3879                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3880                    leaders.len()
3881                )
3882            });
3883            leaders = borda_leaders;
3884            leaders.sort_unstable();
3885        }
3886        let winner = *leaders
3887            .first()
3888            .or(viable.first())
3889            .context("no candidate to declare a winner from")?;
3890
3891        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3892        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3893        let deliberated = !self.state.deliberation.is_empty();
3894
3895        // Whose verdict is this? A rate-limited seat is absent even if it
3896        // ranked before the limit hit, so presence is measured against the
3897        // recorded losses, not just "did a ranking ever appear".
3898        let quota_seats: std::collections::BTreeSet<&str> =
3899            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3900        let mut present = 0usize;
3901        for (i, j) in self.state.judgements.iter().enumerate() {
3902            if quota_seats.contains(j.seat.as_str()) {
3903                continue;
3904            }
3905            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3906            let voted = self
3907                .state
3908                .votes
3909                .iter()
3910                .any(|v| v.judge == i + 1 && v.vote.is_some());
3911            if ranked || voted {
3912                present += 1;
3913            }
3914        }
3915        // Strict majority of the configured panel. A bare majority is real
3916        // signal we can act on, while a minority verdict must never stand in
3917        // for a healthy one. A one-candidate run needs no panel at all, and
3918        // `judges` stays `0` rather than the roster size a panel that never
3919        // sat would otherwise be credited with.
3920        let needs_quorum = viable.len() > 1;
3921        let judges_total = if needs_quorum {
3922            self.roles.judges.len()
3923        } else {
3924            0
3925        };
3926        let quorum = if needs_quorum {
3927            judges_total / 2 + 1
3928        } else {
3929            0
3930        };
3931        let met_quorum = !needs_quorum || present >= quorum;
3932        let uncontested = (!needs_quorum).then(|| {
3933            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3934        });
3935
3936        self.state.event(
3937            "tally",
3938            match &uncontested {
3939                Some(reason) => format!("winner {winner} — {reason}"),
3940                None => format!(
3941                    "winner {winner} — votes {} | initial {} | {} changed | \
3942                     {present}/{judges_total} judges{}",
3943                    first_choice
3944                        .iter()
3945                        .map(|(k, v)| format!("{k}:{v}"))
3946                        .collect::<Vec<_>>()
3947                        .join(" "),
3948                    if unanimous_initial {
3949                        "unanimous"
3950                    } else {
3951                        "split"
3952                    },
3953                    changed_votes,
3954                    if met_quorum {
3955                        String::new()
3956                    } else {
3957                        format!(" — below quorum ({quorum} required)")
3958                    },
3959                ),
3960            },
3961        );
3962        if !met_quorum {
3963            self.state.event(
3964                "stall",
3965                format!(
3966                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3967                     the run stops here, resumable"
3968                ),
3969            );
3970        }
3971        self.state.tally = Some(Tally {
3972            first_choice,
3973            borda,
3974            winner,
3975            rankings: tops.len(),
3976            unanimous_initial,
3977            deliberated,
3978            changed_votes,
3979            unanimous_final,
3980            tie_break,
3981            judges: judges_total,
3982            present,
3983            quorum,
3984            met_quorum,
3985            uncontested,
3986        });
3987        self.state.status = if met_quorum {
3988            RunStatus::Reviewing
3989        } else {
3990            RunStatus::Stalled
3991        };
3992        self.state.save()?;
3993        Ok(())
3994    }
3995
3996    // ------------------------------------------------------------- recover
3997
3998    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3999    /// resumed toward completion once the transient cause clears.
4000    ///
4001    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4002    /// it toward the quorum, which is exactly the set of seats whose absence
4003    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4004    /// not depend on which node happened to hit the limit), or an ordinary
4005    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4006    /// seat is never disturbed.
4007    ///
4008    /// A seat that now answers with a usable ranking is "recovered": its
4009    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4010    /// `tally` counts it present again), and its vote re-collected. A seat that
4011    /// still fails keeps its loss and stays absent.
4012    ///
4013    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4014    /// to review/gate/merge), `false` when it is still below quorum (the run
4015    /// stays `Stalled`, still resumable for a later retry).
4016    #[allow(clippy::too_many_lines)]
4017    async fn recover_stall(&mut self) -> Result<bool> {
4018        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4019        // agent files with `magi task add` name the run that paid for it. The
4020        // prompt overlay is cloned alongside it because the waves borrow it
4021        // while `self` is mutably borrowed by the node's own bookkeeping.
4022        let run_id = self.state.id.clone();
4023        let prompts = self.state.config.prompts.clone();
4024        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4025        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4026        // is what keeps a non-quota collapse — or a quota loss recorded at the
4027        // deliberate node — from being a permanent dead-end on `--resume`.
4028        let quota_seats: BTreeSet<&str> =
4029            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4030        let absent: Vec<String> = self
4031            .state
4032            .judgements
4033            .iter()
4034            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4035            .map(|j| j.seat.clone())
4036            .collect();
4037        if absent.is_empty() {
4038            return Ok(false);
4039        }
4040        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4041        if viable.len() <= 1 {
4042            return Ok(false);
4043        }
4044        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4045        let language = self.state.config.graph.language.clone();
4046        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4047        let sessions = self.state.config.graph.sessions;
4048        let artifacts = agent::artifacts_dir(&self.state.dir());
4049        let root = self.state.worktree_root();
4050        let base_short = short(&self.state.base_commit);
4051        let candidates: Vec<Candidate> = viable.clone();
4052
4053        // Map each absent seat key to its 0-based position in `roles.judges`.
4054        let mut positions: Vec<usize> = absent
4055            .iter()
4056            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4057            .collect();
4058        if positions.is_empty() {
4059            return Ok(false);
4060        }
4061        positions.sort_unstable();
4062        positions.dedup();
4063
4064        // Re-rank the lost seats, one blind prompt each.
4065        let mut judge_jobs = Vec::new();
4066        for &j in &positions {
4067            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4068            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4069            let seat_key = format!("judge-{}", j + 1);
4070            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4071            let seat = self.seat(&seat_key, &spec.id);
4072            judge_jobs.push(SeatJob {
4073                spec,
4074                seat,
4075                prompt: prompt::judge(
4076                    &self.state.instruction,
4077                    &views,
4078                    self.roles.judges.len(),
4079                    &base_short,
4080                    &language,
4081                ),
4082                cwd: root.join(seat_key),
4083                timeout,
4084                allow_write: false,
4085                sessions,
4086                artifacts: artifacts.clone(),
4087                stem: format!("judge-{}-recover", j + 1),
4088                handover: None,
4089            });
4090        }
4091
4092        let labels_for_check = labels.clone();
4093        let mut judge_losses = Vec::new();
4094        let retries = self.state.config.graph.retries;
4095        let cache = self.state.config.cache_dir();
4096        let ctx = WaveCtx {
4097            carry_seats: false,
4098            pauses: Some((&self.pause, &self.interrupt)),
4099            run: &run_id,
4100            node: "judge",
4101            prompts: &prompts,
4102            cache: cache.as_deref(),
4103            round: None,
4104        };
4105        let results = ask_json_wave::<Ranking>(
4106            judge_jobs,
4107            Arc::clone(&self.sem),
4108            retries,
4109            &self.roles.judge_roster,
4110            &ctx,
4111            &mut judge_losses,
4112            &mut self.state,
4113            &move |r: &Ranking| r.validate(&labels_for_check),
4114        )
4115        .await;
4116        if self.state.resource_block.is_some() {
4117            self.stop_for_resource()?;
4118            return Ok(false);
4119        }
4120
4121        // Refresh the judgement of every seat that ranked again.
4122        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4123        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4124            let agent_id = seat.agent.clone();
4125            self.state.seats.insert(seat.key.clone(), seat);
4126            let record = &mut self.state.judgements[j];
4127            match res {
4128                Ok((ranking, out)) => {
4129                    record.agent = agent_id;
4130                    record.ranking = ranking.normalized();
4131                    record.reasons = ranking.reasons;
4132                    record.strengths = ranking.strengths;
4133                    record.confidence = ranking.confidence;
4134                    record.failed = None;
4135                    record.duration_ms = out.duration_ms;
4136                    recovered.insert(j);
4137                    self.state.event(
4138                        "recover",
4139                        format!("judge {} ranked again after the limit", j + 1),
4140                    );
4141                }
4142                Err(e) => {
4143                    self.state
4144                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4145                }
4146            }
4147        }
4148
4149        // Re-ask the votes of the seats that recovered a ranking.
4150        let mut vote_jobs = Vec::new();
4151        let mut vote_pos: Vec<usize> = Vec::new();
4152        for &j in &recovered {
4153            let seat_key = format!("judge-{}", j + 1);
4154            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4155            let seat = self.seat(&seat_key, &spec.id);
4156            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4157            let text = if has_context(&spec, &seat, sessions) {
4158                prompt::final_vote(&labels, &language)
4159            } else {
4160                full.clone()
4161            };
4162            vote_jobs.push(SeatJob {
4163                spec,
4164                seat,
4165                prompt: text,
4166                cwd: root.join(seat_key),
4167                timeout,
4168                allow_write: false,
4169                sessions,
4170                artifacts: artifacts.clone(),
4171                stem: format!("vote-judge-{}-recover", j + 1),
4172                handover: Some(full),
4173            });
4174            vote_pos.push(j);
4175        }
4176        let allowed = labels.clone();
4177        let mut vote_losses = Vec::new();
4178        let vote_retries = self.state.config.graph.retries;
4179        let vote_cache = self.state.config.cache_dir();
4180        let ctx = WaveCtx {
4181            carry_seats: false,
4182            pauses: Some((&self.pause, &self.interrupt)),
4183            run: &run_id,
4184            node: "vote",
4185            prompts: &prompts,
4186            cache: vote_cache.as_deref(),
4187            round: None,
4188        };
4189        let votes = ask_json_wave::<FinalVote>(
4190            vote_jobs,
4191            Arc::clone(&self.sem),
4192            vote_retries,
4193            &self.roles.judge_roster,
4194            &ctx,
4195            &mut vote_losses,
4196            &mut self.state,
4197            &move |v: &FinalVote| match v.label() {
4198                Some(c) if allowed.contains(&c) => Ok(()),
4199                other => bail!("vote {other:?} is not one of {allowed:?}"),
4200            },
4201        )
4202        .await;
4203        if self.state.resource_block.is_some() {
4204            self.stop_for_resource()?;
4205            return Ok(false);
4206        }
4207        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4208            let agent_id = seat.agent.clone();
4209            self.state.seats.insert(seat.key.clone(), seat);
4210            match res {
4211                Ok((v, _)) => {
4212                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4213                        rec.vote = v.label();
4214                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4215                    } else {
4216                        self.state.votes.push(VoteRecord {
4217                            judge: j + 1,
4218                            agent: agent_id,
4219                            vote: v.label(),
4220                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4221                            changed: false,
4222                        });
4223                    }
4224                    self.state.event(
4225                        "recover",
4226                        format!("judge {} voted again after the limit", j + 1),
4227                    );
4228                }
4229                Err(e) => {
4230                    self.state
4231                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4232                }
4233            }
4234        }
4235
4236        // A seat that ranked again is present even if its re-vote failed —
4237        // `tally` falls back to the initial ranking's first choice — so clear
4238        // its quota loss. Seats that still fail keep theirs and stay absent.
4239        let recovered_keys: BTreeSet<String> = recovered
4240            .iter()
4241            .map(|&j| format!("judge-{}", j + 1))
4242            .collect();
4243        self.state
4244            .quota
4245            .retain(|q| !recovered_keys.contains(&q.seat));
4246        // A seat that hit the limit again is a fresh loss, not the old one:
4247        // replace the stale entry so the history stays one-per-seat and the
4248        // daemon can tell this attempt's loss from a previous session's.
4249        for loss in judge_losses.into_iter().chain(vote_losses) {
4250            if recovered_keys.contains(&loss.seat) {
4251                continue;
4252            }
4253            self.state.quota.retain(|q| q.seat != loss.seat);
4254            self.state.quota.push(loss);
4255        }
4256
4257        // Recompute the verdict from the refreshed panel.
4258        self.state.tally = None;
4259        self.tally()?;
4260        Ok(self
4261            .state
4262            .tally
4263            .as_ref()
4264            .map(|t| t.met_quorum)
4265            .unwrap_or(false))
4266    }
4267
4268    // ---------------------------------------------------------------- graft
4269
4270    /// Port the strengths the judges attributed to the losing candidates onto
4271    /// the winner, once, between the tally and the review loop.
4272    ///
4273    /// Best-effort by construction: it returns nothing and fails nothing. A
4274    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4275    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4276    /// the run carries on as if the phase were off. See [`graft`] for the
4277    /// decisions and `AGENTS.md` for the invariants.
4278    ///
4279    /// The one exception is a rollback that could not be confirmed: the
4280    /// rejected commit may still be the winner's tip, so the run is blocked
4281    /// (resumable; the record stays `Running`, which makes a resume retry the
4282    /// restore) instead of reviewing, gating and merging it.
4283    async fn graft(&mut self) -> Result<()> {
4284        let Err(e) = self.graft_inner().await else {
4285            return Ok(());
4286        };
4287        if e.is::<crate::repo_lock::ResourceBlocked>() {
4288            return Err(e);
4289        }
4290        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4291            self.state.status = RunStatus::Blocked;
4292            self.state.event("graft", format!("blocked: {e:#}"));
4293            self.state.save()?;
4294            return Err(e);
4295        }
4296        // An error after the `Running` record was saved may have left the
4297        // fixer's work in the tree: the record must not stay unsettled
4298        // (a later resume would reset past the review fixes), so confirm the
4299        // restore now, or block.
4300        if let Some(rec) = self.state.graft.clone()
4301            && !rec.outcome.settled()
4302        {
4303            if let Some(winner) = self.state.winner().cloned() {
4304                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4305                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4306                        return Err(re);
4307                    }
4308                    self.state.status = RunStatus::Blocked;
4309                    self.state.event("graft", format!("blocked: {re:#}"));
4310                    self.state.save()?;
4311                    return Err(re);
4312                }
4313            }
4314            if let Some(g) = self.state.graft.as_mut() {
4315                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4316            }
4317        }
4318        self.state.event("graft", format!("skipped: {e:#}"));
4319        if let Err(e) = self.state.save() {
4320            tracing::warn!("save after graft: {e:#}");
4321        }
4322        Ok(())
4323    }
4324
4325    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4326    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4327    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4328        let _coordination = self.coordinate(wt).await?;
4329        crate::recovery::restore_repair(wt, pre)
4330            .await
4331            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4332    }
4333
4334    /// A skip is a run event, recorded once however often the node is re-entered.
4335    fn graft_skipped(&mut self, why: &str) {
4336        let message = format!("skipped: {why}");
4337        if !self
4338            .state
4339            .events
4340            .iter()
4341            .any(|e| e.node == "graft" && e.message == message)
4342        {
4343            self.state.event("graft", message);
4344        }
4345    }
4346
4347    async fn graft_inner(&mut self) -> Result<()> {
4348        // A round that was started and never finished (crash, kill): the
4349        // fixer may have left anything in the tree, so put the winner back.
4350        if let Some(rec) = self.state.graft.clone()
4351            && !rec.outcome.settled()
4352        {
4353            // Reviews exist only if the graft ended without its record being
4354            // settled (a swallowed error): the tree has moved on since, and
4355            // a reset to `pre` would discard the review fixes.
4356            if self.state.reviews.is_empty()
4357                && let Some(winner) = self.state.winner().cloned()
4358            {
4359                self.graft_restore(&winner.worktree, &rec.pre).await?;
4360            }
4361            if rec.outcome == GraftOutcome::WaitingForRestore {
4362                self.state.graft = None;
4363                self.state.save()?;
4364            } else {
4365                if let Some(g) = self.state.graft.as_mut() {
4366                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4367                }
4368                self.state.event(
4369                    "graft",
4370                    "rolled back: the previous round was interrupted; the winner is untouched",
4371                );
4372                self.state.save()?;
4373                return Ok(());
4374            }
4375        }
4376        let plan = match graft::plan(&self.state) {
4377            Ok(plan) => plan,
4378            Err(skip) => {
4379                if skip != graft::Skip::AlreadyDone {
4380                    self.graft_skipped(skip.reason());
4381                    self.state.save()?;
4382                }
4383                return Ok(());
4384            }
4385        };
4386        let Some(winner) = self.state.winner().cloned() else {
4387            return Ok(());
4388        };
4389        let wt = winner.worktree.clone();
4390        if !git::is_clean(&wt).await? {
4391            self.graft_skipped("the winner's worktree has uncommitted changes");
4392            self.state.save()?;
4393            return Ok(());
4394        }
4395        let pre = git::rev_parse(&wt, "HEAD").await?;
4396        // Saved before the fixer is called: a stop from here on is repaired
4397        // by the branch above on the next entry.
4398        self.state.graft = Some(GraftRecord {
4399            pre: pre.clone(),
4400            post: None,
4401            lines: None,
4402            limit: None,
4403            agent: None,
4404            outcome: GraftOutcome::Running,
4405            requested: graft::requested(&plan),
4406            adoption: None,
4407            adoption_error: None,
4408        });
4409        self.state.event(
4410            "graft",
4411            format!(
4412                "porting strengths of candidate(s) {} onto {}",
4413                plan.strengths
4414                    .iter()
4415                    .map(|(l, _, _)| l.to_string())
4416                    .collect::<Vec<_>>()
4417                    .join(", "),
4418                plan.winner
4419            ),
4420        );
4421        self.state.save()?;
4422
4423        let fix_prompt = prompt::graft(
4424            &self.state.instruction,
4425            plan.winner,
4426            &plan.strengths,
4427            &self.state.config.graph.language,
4428        );
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4433        let (_, seat, out) = self
4434            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4435                SeatJob {
4436                    prompt: fix_prompt.clone(),
4437                    spec,
4438                    seat,
4439                    cwd: wt.clone(),
4440                    timeout,
4441                    allow_write: true,
4442                    sessions,
4443                    artifacts: artifacts.clone(),
4444                    stem: "graft".to_owned(),
4445                    handover: None,
4446                }
4447            })
4448            .await;
4449        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4450            // Preserve the need to restore earlier fallback edits without
4451            // confusing an unstarted seat with an interrupted invocation.
4452            self.state.graft.as_mut().expect("graft record").outcome =
4453                GraftOutcome::WaitingForRestore;
4454            return self.stop_for_resource();
4455        }
4456        let mut why: Option<String> = None;
4457        match &out {
4458            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4459                why = Some("the fixer produced nothing usable".to_owned());
4460            }
4461            AgentOutcome::Ok(_) => {}
4462            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4463            AgentOutcome::Quota(o) => {
4464                self.state.quota.push(QuotaLoss {
4465                    seat: seat.key.clone(),
4466                    node: "graft".to_owned(),
4467                    at: Timestamp::now(),
4468                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4469                });
4470                why = Some("rate limited (quota)".to_owned());
4471            }
4472            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4473                why = Some(format!(
4474                    "the fixer failed: {}",
4475                    e.lines()
4476                        .next()
4477                        .unwrap_or("")
4478                        .chars()
4479                        .take(160)
4480                        .collect::<String>()
4481                ));
4482            }
4483        }
4484        let agent_id = seat.agent.clone();
4485        self.state.seats.insert(seat.key.clone(), seat);
4486
4487        let mut post = None;
4488        let mut lines = None;
4489        let _coordination = self.coordinate(&wt).await?;
4490        if why.is_none() {
4491            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4492            if let Ok(r) = &rescued {
4493                self.state.note_withheld("graft", &r.withheld);
4494            }
4495            let head = git::rev_parse(&wt, "HEAD").await?;
4496            // What is measured must be what gets reviewed and gated: a failed
4497            // rescue, or anything still uncommitted (withheld lockfiles
4498            // aside, which are never part of the tree), rejects the graft.
4499            let dirty = match &rescued {
4500                Ok(r) => git::status_porcelain(&wt)
4501                    .await
4502                    .map(|st| {
4503                        st.lines().any(|l| {
4504                            !r.withheld
4505                                .iter()
4506                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4507                        })
4508                    })
4509                    .unwrap_or(true),
4510                Err(_) => true,
4511            };
4512            if dirty {
4513                why = Some("the fixer left changes that could not be committed".to_owned());
4514            } else if head == pre {
4515                why = Some("the fixer changed nothing".to_owned());
4516            } else {
4517                lines = git::changed_lines(&wt, &pre, &head).await;
4518                post = Some(head);
4519            }
4520        }
4521        let (adoption, adoption_error) = match &out {
4522            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4523            _ => (None, None),
4524        };
4525        let requested = graft::requested(&plan);
4526        let outcome = match why {
4527            None => {
4528                self.state.event(
4529                    "graft",
4530                    format!(
4531                        "applied: {}",
4532                        lines.map_or_else(
4533                            || "changed lines unavailable".to_owned(),
4534                            |n| format!("{n} changed lines")
4535                        )
4536                    ),
4537                );
4538                GraftOutcome::Applied
4539            }
4540            Some(why) => {
4541                // Not settled until the restore is confirmed: the record
4542                // stays `Running`, so a resume retries it.
4543                crate::recovery::restore_repair(&wt, &pre)
4544                    .await
4545                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4546                self.state.event(
4547                    "graft",
4548                    format!("rolled back: {why}; the winner is untouched"),
4549                );
4550                GraftOutcome::RolledBack(why)
4551            }
4552        };
4553        self.state.graft = Some(GraftRecord {
4554            pre,
4555            post,
4556            lines,
4557            limit: None,
4558            agent: Some(agent_id),
4559            outcome,
4560            requested,
4561            adoption,
4562            adoption_error,
4563        });
4564        self.state.save()?;
4565        Ok(())
4566    }
4567
4568    // ----------------------------------------------------------------- fold
4569
4570    async fn fold_losers(&mut self) -> Result<()> {
4571        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4572            return Ok(());
4573        };
4574        if !self.state.candidate_preservation.is_empty() {
4575            bail!("candidate preservation incomplete; refusing cleanup");
4576        }
4577        let repo = self.state.repo.clone();
4578        let _coordination = self.coordinate(&repo).await?;
4579        let mut folded = Vec::new();
4580        for i in 0..self.state.candidates.len() {
4581            let c = &self.state.candidates[i];
4582            if c.label == winner || c.folded {
4583                continue;
4584            }
4585            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4586            remove_preserved_worktree(&repo, &wt).await?;
4587            if git::branch_exists(&repo, &branch).await?
4588                && !git::branch_delete(&repo, &branch).await?
4589            {
4590                bail!("candidate branch removal refused");
4591            }
4592            self.state.candidates[i].folded = true;
4593            self.state.save()?;
4594            folded.push(label.to_string());
4595        }
4596        // The judges are finished; their checkouts are pure cost from here.
4597        let root = self.state.worktree_root();
4598        for j in 1..=self.roles.judges.len() {
4599            let wt = root.join(format!("judge-{j}"));
4600            if wt.exists() {
4601                remove_preserved_worktree(&repo, &wt).await?;
4602            }
4603        }
4604        // The design-deliberation stage is finished by the time a tally
4605        // exists — same reasoning as the judges above.
4606        if self.state.config.graph.advise {
4607            for k in 1..=self.state.config.graph.advisors {
4608                let wt = root.join(format!("advisor-{k}"));
4609                if wt.exists() {
4610                    remove_preserved_worktree(&repo, &wt).await?;
4611                }
4612            }
4613        }
4614        if !folded.is_empty() {
4615            self.state
4616                .event("fold", format!("folded candidates {}", folded.join(", ")));
4617            self.state.save()?;
4618        }
4619        Ok(())
4620    }
4621
4622    // ------------------------------------------------------------ base sync
4623
4624    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4625    /// anything verifies it.
4626    ///
4627    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4628    /// read whatever is checked out in the winner's worktree. Left alone that
4629    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4630    /// it when the run *branched* - and a run takes long enough that the base
4631    /// has usually moved by the time it gets here. A gate that ran there
4632    /// answers "green on the commit this run started from", not "green on
4633    /// what is about to land", and the difference showed up three times in
4634    /// one day as a green run whose merge would have reverted a file another
4635    /// pull request had already landed.
4636    ///
4637    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4638    /// `land::Step::Rebase` calls, rather than a second implementation of the
4639    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4640    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4641    /// doc warns about ("two answers to a question nobody notices until a
4642    /// diff is wrong").
4643    ///
4644    /// A conflict is not the end of the road: the standing rebase is handed
4645    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4646    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4647    /// land). Once it finishes, review and the gate run as usual on the
4648    /// rebased tree, which is where a breakage the new base caused is caught
4649    /// by the ordinary gate-fix round. magi resolves nothing itself.
4650    ///
4651    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4652    /// `state.base_sync.attempts`, is how many times the base is *rebased
4653    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4654    /// *conflict* was given to a fixer. When the fixer cannot finish the
4655    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4656    /// what was tried (rounds spent, paths still conflicted) and the branch
4657    /// and worktree stay exactly as they were - untouched, for a person to
4658    /// look at - which is also what makes re-entering this function
4659    /// afterwards a no-op instead of a second attempt at the same wall. A
4660    /// push failure ends the same way.
4661    async fn sync_to_base(&mut self) -> Result<()> {
4662        if self.state.status == RunStatus::AlreadyInBase {
4663            return Ok(());
4664        }
4665        let conflicted = self
4666            .state
4667            .base_sync
4668            .as_ref()
4669            .is_some_and(|s| s.conflict.is_some());
4670        let Some(winner) = self.state.winner().cloned() else {
4671            return Ok(());
4672        };
4673
4674        let repo = self.state.repo.clone();
4675        let remote = self.state.config.merge.remote.clone();
4676        let base_branch = self.state.base_branch.clone();
4677        let tracking = format!("{remote}/{base_branch}");
4678
4679        git::fetch(&repo, &remote, &base_branch).await.ok();
4680        // No network, or the remote never had this branch: the run already
4681        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4682        // otherwise), and one that got this far is not blocked by a fetch
4683        // that fails now. It just has no newer tip to compare against.
4684        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4685            return Ok(());
4686        };
4687
4688        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4689        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4690        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4691
4692        // Before any rebase, and before a recorded conflict is honoured: a
4693        // branch whose change reached the base under other commit ids has
4694        // nothing to rebase and nothing to conflict with, and a run that
4695        // already stopped on that phantom conflict recovers here on resume.
4696        // `behind == 0` with head == tip is a branch the base has since taken
4697        // in whole, whether or not a conflict was ever recorded: the ancestry
4698        // proof must still run (`classify` ignores a head still on the start
4699        // commit).
4700        if (behind > 0 || conflicted || head == tip)
4701            && self
4702                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4703                .await?
4704        {
4705            return Ok(());
4706        }
4707        if conflicted {
4708            return Ok(());
4709        }
4710
4711        if behind == 0 {
4712            // A fixer-finished rebase moves the branch ref before the
4713            // winner's worktree is told (`sync_to_head` below). A run that
4714            // died in between resumes here with `behind == 0` and a tree still
4715            // holding the pre-rebase files, which review and the gate would
4716            // then read. That state is exactly: HEAD moved off the tip the
4717            // rebase started from, yet the tree is still identical to that
4718            // tip. A tree with edits of its own differs from it, so nothing
4719            // is thrown away.
4720            if let Some(from) = self
4721                .state
4722                .rebase_fixes
4723                .iter()
4724                .rev()
4725                .find_map(|r| r.from.clone())
4726                && from != head
4727                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4728                    .await
4729                    .is_ok_and(|o| o.ok())
4730            {
4731                git::sync_to_head(&winner.worktree).await?;
4732            }
4733            // An earlier attempt may have rebased the branch locally and died
4734            // before pushing it (only the fresh-rebase arm below pushes).
4735            // Publish it now, so the plain push at PR time is not refused as
4736            // a non-fast-forward. A run already holding a recorded conflict
4737            // never reaches here; that case is out of scope.
4738            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4739            if let Some(why) = &conflict {
4740                self.state.status = RunStatus::Blocked;
4741                self.state.event("land", why.clone());
4742            }
4743            self.state.base_sync = Some(BaseSync {
4744                tip,
4745                behind: 0,
4746                attempts,
4747                conflict,
4748                already_in: None,
4749            });
4750            self.state.save()?;
4751            return Ok(());
4752        }
4753
4754        if attempts >= BASE_SYNC_ROUNDS {
4755            let why = format!(
4756                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4757                 rebase(s); rebasing again would only race it",
4758                winner.branch
4759            );
4760            self.state.status = RunStatus::Blocked;
4761            self.state.base_sync = Some(BaseSync {
4762                tip,
4763                behind,
4764                attempts,
4765                conflict: Some(why.clone()),
4766                already_in: None,
4767            });
4768            self.state.event("land", why);
4769            self.state.save()?;
4770            return Ok(());
4771        }
4772
4773        self.state.event(
4774            "land",
4775            format!(
4776                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4777                winner.branch
4778            ),
4779        );
4780        self.state.save()?;
4781
4782        // The remote's copy of the branch, read now and only if the fetch
4783        // really succeeded (a stale tracking ref must never pin a lease). It is
4784        // pushed over after a rebase only when it is a commit this branch
4785        // already contains, by ancestry or by patch (an earlier rebase of ours
4786        // that never reached the remote): anything else is somebody else's work.
4787        let branch_tracking = format!("{remote}/{}", winner.branch);
4788        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4789        let remote_tip = if self.state.task_pr_branch().is_none()
4790            && matches!(&fetched_branch, Ok(o) if o.ok())
4791        {
4792            git::rev_parse(&repo, &branch_tracking).await.ok()
4793        } else {
4794            None
4795        };
4796        // A remote tip this branch does not contain is somebody else's work:
4797        // rebasing would leave a local tip that can never be pushed. Stop
4798        // before touching anything and say so.
4799        if let Some(theirs) = &remote_tip
4800            && !git::is_ancestor(&repo, theirs, &head).await
4801            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4802                .await
4803                .is_ok_and(|missing| missing.is_empty())
4804        {
4805            let why = format!(
4806                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4807                 them",
4808                short(theirs),
4809                winner.branch
4810            );
4811            self.state.status = RunStatus::Blocked;
4812            self.state.base_sync = Some(BaseSync {
4813                tip,
4814                behind,
4815                attempts,
4816                conflict: Some(why.clone()),
4817                already_in: None,
4818            });
4819            self.state.event("land", why);
4820            self.state.save()?;
4821            return Ok(());
4822        }
4823
4824        let scratch = self.state.dir().join("base-sync");
4825        let rebased = match crate::rebase::rebase_with_fixer(
4826            &mut self.state,
4827            &scratch,
4828            &winner.branch,
4829            &tracking,
4830        )
4831        .await
4832        {
4833            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4834            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4835            Err(e) => Err(e),
4836        };
4837        let attempts = attempts + 1;
4838        match rebased {
4839            Ok(None) => {
4840                // The branch ref moved, but a worktree that already had it
4841                // checked out (the winner's) was not told; sync its index and
4842                // files before anything reads them.
4843                git::sync_to_head(&winner.worktree).await?;
4844                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4845                let mut conflict = None;
4846                if let Some(pinned) = &remote_tip {
4847                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4848                    match pushed {
4849                        Ok(o) if o.ok() => self.state.event(
4850                            "land",
4851                            format!("pushed rebased {} to {remote}", winner.branch),
4852                        ),
4853                        Ok(o) => {
4854                            conflict = Some(format!(
4855                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4856                                winner.branch,
4857                                short(pinned),
4858                                o.stderr.chars().take(600).collect::<String>()
4859                            ));
4860                        }
4861                        Err(e) => {
4862                            conflict = Some(format!(
4863                                "rebased {} locally but could not push it: {e:#}",
4864                                winner.branch
4865                            ));
4866                        }
4867                    }
4868                }
4869                if let Some(why) = &conflict {
4870                    self.state.status = RunStatus::Blocked;
4871                    self.state.event("land", why.clone());
4872                }
4873                self.state.base_sync = Some(BaseSync {
4874                    tip: tip.clone(),
4875                    behind: 0,
4876                    attempts,
4877                    conflict,
4878                    already_in: None,
4879                });
4880                self.state
4881                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4882            }
4883            Ok(Some(conflict)) => {
4884                let why = format!(
4885                    "{} conflicts with {tracking} and did not rebase: {}",
4886                    winner.branch,
4887                    conflict.chars().take(600).collect::<String>()
4888                );
4889                self.state.status = RunStatus::Blocked;
4890                self.state.base_sync = Some(BaseSync {
4891                    tip,
4892                    behind,
4893                    attempts,
4894                    conflict: Some(why.clone()),
4895                    already_in: None,
4896                });
4897                self.state.event("land", why);
4898            }
4899            // A refused Git write or a busy repository: nothing is wrong with
4900            // the branch. The scratch rebase stays standing and the run
4901            // resumes into it (`base_sync` must not claim a conflict).
4902            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4903                self.state.save()?;
4904                return Ok(());
4905            }
4906            Err(e) => {
4907                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4908                self.state.status = RunStatus::Blocked;
4909                self.state.base_sync = Some(BaseSync {
4910                    tip,
4911                    behind,
4912                    attempts,
4913                    conflict: Some(why.clone()),
4914                    already_in: None,
4915                });
4916                self.state.event("land", why);
4917            }
4918        }
4919        self.state.save()?;
4920        Ok(())
4921    }
4922
4923    /// Push a branch an earlier attempt rebased locally but never published,
4924    /// pinned to the remote tip read right after a successful fetch. Returns
4925    /// the reason when the run must stop; `None` when there was nothing to do
4926    /// (no remote copy, the same tip, or a remote copy this branch already
4927    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4928    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4929        // Task competitions publish only after the PR text gate, to their
4930        // public head; never resurrect a candidate branch on the remote.
4931        if self.state.task_pr_branch().is_some() {
4932            return None;
4933        }
4934        let repo = self.state.repo.clone();
4935        let remote = self.state.config.merge.remote.clone();
4936        let fetched = git::fetch(&repo, &remote, branch).await;
4937        if !matches!(&fetched, Ok(o) if o.ok()) {
4938            return None;
4939        }
4940        let branch_tracking = format!("{remote}/{branch}");
4941        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4942        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4943            return None;
4944        }
4945        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4946            .await
4947            .is_ok_and(|missing| missing.is_empty())
4948        {
4949            return Some(format!(
4950                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4951                 them",
4952                short(&theirs)
4953            ));
4954        }
4955        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4956            Ok(o) if o.ok() => {
4957                self.state
4958                    .event("land", format!("pushed rebased {branch} to {remote}"));
4959                None
4960            }
4961            Ok(o) => Some(format!(
4962                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4963                 someone may have pushed): {}",
4964                short(&theirs),
4965                o.stderr.chars().take(600).collect::<String>()
4966            )),
4967            Err(e) => Some(format!(
4968                "{branch} is rebased locally but could not be pushed: {e:#}"
4969            )),
4970        }
4971    }
4972
4973    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4974    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4975    /// whether it did.
4976    ///
4977    /// Checked only when the base is ahead of the branch. A failing check is
4978    /// "not proven" - the ordinary rebase path then decides - never a reason to
4979    /// stop the run.
4980    ///
4981    /// The remote copy of the branch is held to the same standard as the local
4982    /// one: if it carries a tip this worktree does not, that tip must itself be
4983    /// proven in the base, or nothing is settled (a pull request would
4984    /// otherwise be closed over commits nobody checked). The pull request is
4985    /// closed *before* the terminal status is saved; if that fails for a
4986    /// reason other than a refusal (no network, a `gh` error) the run is left
4987    /// `Blocked` with the reason as its conflict, which a resume retries -
4988    /// the same recovery a phantom conflict gets.
4989    async fn settle_already_in(
4990        &mut self,
4991        branch: &str,
4992        tip: &str,
4993        head: &str,
4994        attempts: usize,
4995        behind: usize,
4996    ) -> Result<bool> {
4997        let repo = self.state.repo.clone();
4998        let remote = self.state.config.merge.remote.clone();
4999        let start = self.state.base_commit.clone();
5000        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5001            Ok(Some(e)) => e,
5002            Ok(None) => return Ok(false),
5003            Err(e) => {
5004                tracing::warn!("already-in-base check for {branch}: {e:#}");
5005                return Ok(false);
5006            }
5007        };
5008        let mut verified = vec![head.to_owned()];
5009        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5010        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5011        if matches!(&fetched, Ok(o) if o.ok())
5012            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5013            && theirs != head
5014        {
5015            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5016                Ok(Some(_)) => verified.push(theirs),
5017                _ => return Ok(false),
5018            }
5019        }
5020        let base_branch = self.state.base_branch.clone();
5021        let message = format!(
5022            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5023             land",
5024            evidence.names(),
5025            evidence.proof.as_str()
5026        );
5027        let closed =
5028            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5029                .await;
5030        match closed {
5031            Ok(Ok(url)) => self
5032                .state
5033                .event("land", format!("closed {url}: superseded on {base_branch}")),
5034            Ok(Err(why)) => self
5035                .state
5036                .event("land", format!("did not close a pull request: {why}")),
5037            Err(e) => {
5038                let why = format!(
5039                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5040                     not be closed ({e:#}); resume to retry"
5041                );
5042                self.state.status = RunStatus::Blocked;
5043                self.state.base_sync = Some(BaseSync {
5044                    tip: tip.to_owned(),
5045                    behind,
5046                    attempts,
5047                    conflict: Some(why.clone()),
5048                    already_in: None,
5049                });
5050                self.state.event("land", why);
5051                self.state.save()?;
5052                return Ok(true);
5053            }
5054        }
5055        self.state.status = RunStatus::AlreadyInBase;
5056        self.state.base_sync = Some(BaseSync {
5057            tip: tip.to_owned(),
5058            behind,
5059            attempts,
5060            conflict: None,
5061            already_in: Some(evidence),
5062        });
5063        self.state.event("land", message);
5064        self.state.save()?;
5065        self.settle_questions();
5066        Ok(true)
5067    }
5068
5069    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5070    /// last landed the winner on, once it has run, else the commit the run
5071    /// branched from.
5072    ///
5073    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5074    /// and `vote` all happen before there is a winner to rebase, so they
5075    /// compare every candidate against the branch point on purpose, and a
5076    /// base that moves after they are already done cannot change an answer
5077    /// they already gave.
5078    fn landing_base(&self) -> String {
5079        self.state
5080            .base_sync
5081            .as_ref()
5082            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5083    }
5084
5085    // ------------------------------------------------------- operator fix
5086
5087    /// Route specific, already-recorded review findings to a fixer for a
5088    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5089    /// entry point.
5090    ///
5091    /// Distinct from `review_loop`'s own fix step in three ways: it never
5092    /// runs a reviewer wave, it never spends review-round budget, and what
5093    /// happened is recorded as an [`OperatorFixRequest`] appended to
5094    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5095    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5096    /// and vote must never be rewritten to look like a manufactured blocking
5097    /// verdict.
5098    ///
5099    /// Only meaningful once review has actually concluded: `Ready` (handed
5100    /// off with findings still open, or simply concluded clean while minor
5101    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5102    /// gate failed). Everything else is refused: a run still in progress
5103    /// should simply be resumed, and a `Merged` run's branch has already
5104    /// landed — reopening *this* run's own record cannot change that, so the
5105    /// answer there is a fresh `magi review <branch>`.
5106    ///
5107    /// A real commit here re-verifies through a fresh, ordinary review-only
5108    /// run on the same branch ([`Self::review`]) rather than reopening this
5109    /// run's own `review_loop`: once any round in this run's history went
5110    /// clean, `review_conclusion` treats that as permanent by design (the
5111    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5112    /// way to force one more genuine reviewer wave out of *this* run without
5113    /// either rewriting history or weakening that guarantee for every other
5114    /// caller. A review-only run costs nothing extra — no implementation, no
5115    /// judging, no vote — and exercises the exact same review → verify →
5116    /// gate → (human) merge path, unmodified.
5117    pub async fn fix_selected(
5118        &mut self,
5119        ids: &[String],
5120        reason: &str,
5121        allow_stale: bool,
5122    ) -> Result<()> {
5123        let reason = reason.trim();
5124        if reason.is_empty() {
5125            bail!("a fix request needs a reason — that is the operator's own record of why");
5126        }
5127        if ids.is_empty() {
5128            bail!("no finding id given");
5129        }
5130        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5131            bail!(
5132                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5133                 has already concluded — can be given a targeted fix. A run still \
5134                 in progress should simply be resumed; a `merged` run's branch has \
5135                 already landed, so its answer is a fresh `magi review <branch>`, \
5136                 not reopening this run's own record",
5137                self.state.id,
5138                self.state.status.as_str()
5139            );
5140        }
5141        let Some(winner) = self.state.winner().cloned() else {
5142            bail!("run {} has no winning candidate to fix", self.state.id);
5143        };
5144        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5145            bail!(
5146                "branch `{}` no longer exists; this run cannot be extended",
5147                winner.branch
5148            );
5149        }
5150        let home = crate::run::home();
5151        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5152            bail!(
5153                "run {} is currently being worked on by another magi process",
5154                self.state.id
5155            );
5156        }
5157        // Held for the rest of this call, including the follow-up review
5158        // below: two `magi fix` invocations against the same run must not
5159        // both reach the worktree manipulation further down, which would
5160        // otherwise race to remove and recreate the same directory — see
5161        // [`FixClaim`]'s own doc.
5162        let _claim = FixClaim::acquire(&self.state.dir())?;
5163
5164        // Resolve every id before spending anything — an unknown id refuses
5165        // the whole request rather than silently dropping it — and dedup
5166        // while keeping the operator's own order.
5167        let mut seen = BTreeSet::new();
5168        let mut findings = Vec::new();
5169        let mut missing = Vec::new();
5170        for id in ids {
5171            if !seen.insert(id.clone()) {
5172                continue;
5173            }
5174            match self.state.finding(id) {
5175                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5176                    id: f.id.clone(),
5177                    severity: f.severity,
5178                    reviewer_vote: rec.vote,
5179                    round: round.round,
5180                    round_head: round.head.clone(),
5181                    reviewer: rec.reviewer,
5182                    agent: rec.agent.clone(),
5183                    file: f.file.clone(),
5184                    line: f.line,
5185                    title: f.title.clone(),
5186                    detail: f.detail.clone(),
5187                    outcome: OperatorFixOutcome::Pending,
5188                }),
5189                None => missing.push(id.clone()),
5190            }
5191        }
5192        if !missing.is_empty() {
5193            bail!(
5194                "unknown finding id(s): {}; nothing was changed",
5195                missing.join(", ")
5196            );
5197        }
5198
5199        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5200        let stale_details: Vec<(String, String)> = findings
5201            .iter()
5202            .filter(|f| f.round_head != head_at_request)
5203            .map(|f| (f.id.clone(), f.round_head.clone()))
5204            .collect();
5205        let stale = !stale_details.is_empty();
5206        if stale && !allow_stale {
5207            bail!(
5208                "the branch has moved since some finding(s) were raised — {} — now \
5209                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5210                stale_details
5211                    .iter()
5212                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5213                    .collect::<Vec<_>>()
5214                    .join(", "),
5215                short(&head_at_request)
5216            );
5217        }
5218
5219        let request = OperatorFixRequest {
5220            requested_at: Timestamp::now(),
5221            reason: reason.to_owned(),
5222            findings,
5223            head_at_request: head_at_request.clone(),
5224            allow_stale,
5225            stale,
5226            fix: None,
5227            result_head: None,
5228            follow_up_review_run: None,
5229        };
5230        self.state.event(
5231            "fix",
5232            format!(
5233                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5234                request.findings.len(),
5235                request
5236                    .findings
5237                    .iter()
5238                    .map(|f| f.id.as_str())
5239                    .collect::<Vec<_>>()
5240                    .join(", "),
5241            ),
5242        );
5243        // Recorded now, before any worktree work or the fixer call itself —
5244        // and re-saved at each checkpoint below: a crash at any point after
5245        // this (mid fixer call, mid follow-up review) must not lose the fact
5246        // that this was requested, for which findings, and why. Everything
5247        // past this point reads and writes through `request_index` rather
5248        // than a local variable, since `request` itself is moved here.
5249        self.state.operator_fixes.push(request);
5250        self.state.save()?;
5251        let request_index = self.state.operator_fixes.len() - 1;
5252
5253        // A fresh, dedicated worktree for this one call, never the winner's
5254        // own worktree in place: that one may already be gone (folded away),
5255        // and reusing it in place would leave the branch checked out there
5256        // when the follow-up review below tries to check it out again. Freed
5257        // immediately after, either way — but only once confirmed clean:
5258        // `worktree_remove` is a `git worktree remove --force`, which would
5259        // otherwise discard uncommitted work left there by the operator or
5260        // another process before this had a chance to even look at it.
5261        let repo = self.state.repo.clone();
5262        let coordination = self.coordinate(&repo).await?;
5263        if winner.worktree.exists() {
5264            // Lockfiles a rescue commit withheld stay untracked on purpose and
5265            // are already recorded; they are not the operator's work to protect.
5266            let dirty = git::git(
5267                &winner.worktree,
5268                &["status", "--porcelain", "--untracked-files=all"],
5269            )
5270            .await?;
5271            let only_withheld = dirty.lines().all(|l| {
5272                l.strip_prefix("?? ")
5273                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5274            });
5275            if !only_withheld {
5276                bail!(
5277                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5278                     discard them first",
5279                    winner.worktree.display()
5280                );
5281            }
5282            git::worktree_remove(&self.state.repo, &winner.worktree)
5283                .await
5284                .ok();
5285        }
5286        let fix_worktree = self.state.worktree_root().join("operator-fix");
5287        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5288        git::git(
5289            &self.state.repo,
5290            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5291        )
5292        .await
5293        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5294        if !git::is_clean(&fix_worktree).await? {
5295            git::worktree_remove(&self.state.repo, &fix_worktree)
5296                .await
5297                .ok();
5298            bail!(
5299                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5300                winner.branch
5301            );
5302        }
5303        let repo_for_setup = self.state.repo.clone();
5304        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5305            Ok(()) => {
5306                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5307            }
5308            Err(e) => Err(e),
5309        };
5310        if let Err(e) = setup {
5311            git::worktree_remove(&self.state.repo, &fix_worktree)
5312                .await
5313                .ok();
5314            return Err(e);
5315        }
5316
5317        let run_id = self.state.id.clone();
5318        let prompts = self.state.config.prompts.clone();
5319        let language = self.state.config.graph.language.clone();
5320        let sessions = self.state.config.graph.sessions;
5321        let artifacts = agent::artifacts_dir(&self.state.dir());
5322        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5323            .findings
5324            .iter()
5325            .map(|f| Finding {
5326                id: f.id.clone(),
5327                severity: f.severity,
5328                file: f.file.clone(),
5329                line: f.line,
5330                title: f.title.clone(),
5331                detail: f.detail.clone(),
5332            })
5333            .collect();
5334        let fix_prompt = prompt::operator_fix_in(
5335            &self.state.instruction,
5336            &finding_list,
5337            reason,
5338            &stale_details,
5339            &head_at_request,
5340            &language,
5341            &self.state.config.graph.github_language,
5342        );
5343        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5344        drop(coordination);
5345        let (job, seat, out) = self
5346            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5347                prompt: fix_prompt.clone(),
5348                spec,
5349                seat,
5350                cwd: fix_worktree.clone(),
5351                timeout,
5352                allow_write: true,
5353                sessions,
5354                artifacts: artifacts.clone(),
5355                stem: "operator-fix".to_owned(),
5356                handover: None,
5357            })
5358            .await;
5359        let agent_id = seat.agent.clone();
5360
5361        let mut fix = FixRecord {
5362            agent: agent_id,
5363            addressed: Vec::new(),
5364            rejected: Vec::new(),
5365            notes: String::new(),
5366            committed: false,
5367            failed: None,
5368            duration_ms: 0,
5369            continuation: None,
5370        };
5371        let mut final_seat = seat.clone();
5372        match out {
5373            AgentOutcome::Ok(o) => {
5374                fix.duration_ms = o.duration_ms;
5375                let parsed = verdict::extract_json::<FixReport>(&o.text);
5376                let incomplete_reason = match &parsed {
5377                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5378                        "the reply parsed, but it reported a command whose own CLI \
5379                         never confirmed an exit status"
5380                            .to_owned(),
5381                    ),
5382                    Ok(_) => None,
5383                    Err(e) => Some(e.to_string()),
5384                };
5385                match incomplete_reason {
5386                    None => {
5387                        let report = parsed.expect("checked Ok above");
5388                        fix.addressed = report.addressed;
5389                        fix.rejected = report.rejected;
5390                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5391                    }
5392                    Some(reason) => {
5393                        let (resumed_seat, resolved, failure, cont) = self
5394                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5395                            .await;
5396                        fix.duration_ms += cont.cumulative_wait_ms;
5397                        fix.continuation = Some(cont);
5398                        final_seat = resumed_seat;
5399                        match resolved {
5400                            Some(report) => {
5401                                fix.addressed = report.addressed;
5402                                fix.rejected = report.rejected;
5403                                fix.notes =
5404                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5405                            }
5406                            None => fix.failed = failure,
5407                        }
5408                    }
5409                }
5410            }
5411            AgentOutcome::Dropped(o) => {
5412                fix.duration_ms = o.duration_ms;
5413                let why = o
5414                    .dropped
5415                    .as_ref()
5416                    .map(|d| d.why.as_str())
5417                    .unwrap_or("the CLI ended the stream without delivering its answer");
5418                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5419            }
5420            AgentOutcome::Quota(o) => {
5421                self.state.quota.push(QuotaLoss {
5422                    seat: final_seat.key.clone(),
5423                    node: "fix".to_owned(),
5424                    at: Timestamp::now(),
5425                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5426                });
5427                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5428            }
5429            AgentOutcome::Failed(e) => fix.failed = Some(e),
5430            AgentOutcome::ResourceBlocked(e) => {
5431                fix.failed = Some(format!("the fixer was not started: {e}"));
5432            }
5433        }
5434        if fix.continuation.is_none() {
5435            fix.continuation = Some(ContinuationRecord::not_needed());
5436        }
5437        self.state.seats.insert(final_seat.key.clone(), final_seat);
5438
5439        let rescue_message = format!(
5440            "magi: operator-selected fix ({}) (uncommitted work)",
5441            self.state.operator_fixes[request_index]
5442                .findings
5443                .iter()
5444                .map(|f| f.id.as_str())
5445                .collect::<Vec<_>>()
5446                .join(", ")
5447        );
5448        let repo = self.state.repo.clone();
5449        let coordination = self.coordinate(&repo).await?;
5450        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5451            self.state.note_withheld("fix", &r.withheld);
5452        }
5453        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5454        fix.committed = after != head_at_request;
5455        git::worktree_remove(&self.state.repo, &fix_worktree)
5456            .await
5457            .ok();
5458        drop(coordination);
5459
5460        self.state.event(
5461            "fix",
5462            match &fix.failed {
5463                Some(reason) => format!(
5464                    "operator fix: adoption report was lost ({reason}); {}",
5465                    if fix.committed {
5466                        "committed"
5467                    } else {
5468                        "NO new commit"
5469                    }
5470                ),
5471                None => format!(
5472                    "operator fix: {} addressed, {} rejected, {}",
5473                    fix.addressed.len(),
5474                    fix.rejected.len(),
5475                    if fix.committed {
5476                        "committed"
5477                    } else {
5478                        "NO new commit"
5479                    }
5480                ),
5481            },
5482        );
5483
5484        // Every selected finding gets an outcome — never left `Pending` once
5485        // the fixer's own turn is over. A report that never came back at all
5486        // marks every one of them `Unreported`, not silently "not addressed":
5487        // quota, a dropped stream, or an exhausted continuation are gaps in
5488        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5489        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5490        for f in &mut self.state.operator_fixes[request_index].findings {
5491            f.outcome = if fix.failed.is_some() {
5492                OperatorFixOutcome::Unreported
5493            } else if fix.addressed.contains(&f.id) {
5494                OperatorFixOutcome::Addressed
5495            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5496                OperatorFixOutcome::Rejected { why: r.why.clone() }
5497            } else {
5498                OperatorFixOutcome::Unreported
5499            };
5500        }
5501
5502        let committed = fix.committed;
5503        if committed {
5504            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5505        }
5506        self.state.operator_fixes[request_index].fix = Some(fix);
5507        // Saved again now that the fixer's own outcome is final, on top of
5508        // the save right after the request was first pushed above.
5509        self.state.save()?;
5510
5511        if committed {
5512            self.state.event(
5513                "fix",
5514                format!(
5515                    "operator fix committed {}; opening a follow-up review-only run",
5516                    short(&after)
5517                ),
5518            );
5519            // The operator asked for the fix, and the follow-up serves whatever
5520            // task the run it follows served.
5521            let origin =
5522                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5523            match Self::review(
5524                &self.state.repo,
5525                &winner.branch,
5526                self.state.config.clone(),
5527                origin,
5528            )
5529            .await
5530            {
5531                Ok(mut follow_up) => {
5532                    follow_up.state.event(
5533                        "start",
5534                        format!(
5535                            "requested by an operator fix on run {} for finding(s) {}",
5536                            self.state.id,
5537                            self.state.operator_fixes[request_index]
5538                                .findings
5539                                .iter()
5540                                .map(|f| f.id.as_str())
5541                                .collect::<Vec<_>>()
5542                                .join(", "),
5543                        ),
5544                    );
5545                    follow_up.state.save()?;
5546                    let follow_up_id = follow_up.state.id.clone();
5547                    // The follow-up is a run like any other: it belongs to the
5548                    // task of the run it follows, or to one filed for it.
5549                    let adopted = match crate::direct::adopt(
5550                        &follow_up.state,
5551                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5552                    ) {
5553                        Ok(a) => a,
5554                        Err(e) => {
5555                            // An ownerless run must not spend agent calls; it
5556                            // stays saved, and `magi run --resume` adopts it.
5557                            self.state.event(
5558                                "fix",
5559                                format!(
5560                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5561                                ),
5562                            );
5563                            self.state.save()?;
5564                            return Ok(());
5565                        }
5566                    };
5567                    let executed = follow_up.execute().await;
5568                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5569                    if let Some(a) = adopted {
5570                        a.finish(&follow_up.state, executed);
5571                    }
5572                    if let Some(e) = failure {
5573                        self.state.event(
5574                            "fix",
5575                            format!(
5576                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5577                            ),
5578                        );
5579                    }
5580                    self.state.operator_fixes[request_index].follow_up_review_run =
5581                        Some(follow_up_id);
5582                }
5583                Err(e) => {
5584                    self.state.event(
5585                        "fix",
5586                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5587                    );
5588                }
5589            }
5590            self.state.save()?;
5591        }
5592
5593        Ok(())
5594    }
5595
5596    // --------------------------------------------------------------- review
5597
5598    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5599    /// names, in order, each at most once, moving on only when the call
5600    /// advances (an error, a quota hit or nothing usable - the same decision
5601    /// point `agent::chain_advances` is for the other chained roles).
5602    ///
5603    /// Each agent gets its own seat from `Runner::seat`: the same agent
5604    /// continues its conversation (the winner's own implementer seat when it
5605    /// is the winner's author, now that the competition is over), another
5606    /// takes a fresh one so the full prompt is sent again. A handover is
5607    /// recorded under `node`, which is what makes the fallback stick for the
5608    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5609    /// is returned, so an exhausted chain reads exactly like a single failed
5610    /// fixer: one quota loss, the same wording, the same refund. An earlier
5611    /// attempt's edits are left in the tree and judged, with the final
5612    /// attempt's, by what git says afterwards.
5613    ///
5614    /// Returns the job that produced the outcome, for `continue_fix_report`.
5615    async fn ask_fixer(
5616        &mut self,
5617        winner: &Candidate,
5618        node: &'static str,
5619        round: Option<usize>,
5620        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5621    ) -> (SeatJob, SeatState, AgentOutcome) {
5622        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5623        self.ask_fixer_among(attempts, node, round, None, build)
5624            .await
5625    }
5626
5627    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5628    /// worktree and a commit: before the next agent is asked after one that
5629    /// did not satisfy the call, the worktree is put back on that commit, so
5630    /// a failed agent's partial edits are not carried into the next one.
5631    async fn ask_fixer_among(
5632        &mut self,
5633        attempts: Vec<(AgentSpec, String)>,
5634        node: &'static str,
5635        round: Option<usize>,
5636        restore: Option<(&Path, &str)>,
5637        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5638    ) -> (SeatJob, SeatState, AgentOutcome) {
5639        let run_id = self.state.id.clone();
5640        let prompts = self.state.config.prompts.clone();
5641        let cache = self.state.config.cache_dir();
5642        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5643        let mut last = None;
5644        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5645            let seat = self.seat(&key, &spec.id);
5646            let mut job = build(spec.clone(), seat);
5647            if i > 0 {
5648                job.stem = format!("{}-{}", job.stem, spec.id);
5649            }
5650            let ctx = WaveCtx {
5651                carry_seats: false,
5652                pauses: Some((&self.pause, &self.interrupt)),
5653                run: &run_id,
5654                node,
5655                prompts: &prompts,
5656                cache: cache.as_deref(),
5657                round,
5658            };
5659            let (seat, out) =
5660                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5661            // A resource block never advances the chain: the chosen agent
5662            // was not asked, so it must be the one asked again on resume.
5663            let advances = match &out {
5664                AgentOutcome::Ok(o) => agent::output_advances(o),
5665                AgentOutcome::ResourceBlocked(_) => false,
5666                _ => true,
5667            };
5668            if let Some(next) = ids.get(i + 1)
5669                && advances
5670            {
5671                self.state.seats.insert(seat.key.clone(), seat.clone());
5672                let class =
5673                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5674                let reason = match &out {
5675                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5676                    other => fail_reason(other),
5677                };
5678                // A failed agent's edits must not reach the next one, and a
5679                // restore that did not work ends the chain: the call fails
5680                // and the caller's own (confirmed) restore decides.
5681                if let Some((wt, rev)) = restore
5682                    && let Err(e) = self.graft_restore(wt, rev).await
5683                {
5684                    let why =
5685                        format!("could not restore the worktree before the next fixer: {e:#}");
5686                    tracing::warn!("{why}");
5687                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5688                        AgentOutcome::ResourceBlocked(why)
5689                    } else {
5690                        AgentOutcome::Failed(why)
5691                    };
5692                    last = Some((job, seat, out));
5693                    break;
5694                }
5695                record_handover(
5696                    &mut self.state,
5697                    node,
5698                    &seat.key,
5699                    &spec.id,
5700                    next,
5701                    &class,
5702                    &reason,
5703                );
5704                continue;
5705            }
5706            last = Some((job, seat, out));
5707            break;
5708        }
5709        last.expect("the fixer chain always has an entry")
5710    }
5711
5712    /// The one repair a verification-only failure earns (see
5713    /// [`crate::recovery`]), before the first review round of a run that took
5714    /// a branch over.
5715    ///
5716    /// Saved as started *before* the fixer is called, so a resume never
5717    /// repairs twice; a start found on re-entry is rolled back to the commit
5718    /// it began from. A failed or unusable fixer leaves the tree as it was and
5719    /// the review goes on over it, so the first round's own verification
5720    /// reports the failure again. The failure is never carried into a round as
5721    /// a result: verification is re-run on whatever commit this leaves.
5722    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5723        use crate::recovery::{Mode, Repair};
5724        if !self.state.reviews.is_empty() {
5725            return Ok(());
5726        }
5727        let Some(mut rec) = self.state.recovery.clone() else {
5728            return Ok(());
5729        };
5730        if rec.mode != Mode::VerifyRepair {
5731            return Ok(());
5732        }
5733        let wt = winner.worktree.clone();
5734        let mut head = git::rev_parse(&wt, "HEAD").await?;
5735        match rec.repair.clone() {
5736            Repair::Done { .. } => return Ok(()),
5737            Repair::WaitingForRestore { before } => {
5738                self.graft_restore(&wt, &before).await?;
5739                head = before;
5740                rec.repair = Repair::NotStarted;
5741                self.state.recovery = Some(rec.clone());
5742                self.state.save()?;
5743            }
5744            Repair::Started { before } => {
5745                // Interrupted: the unfinished edits are not trusted.
5746                // An unconfirmed rollback blocks the run with `Started` kept,
5747                // so a resume retries it; the tree is never reviewed as is.
5748                if let Err(e) = self.graft_restore(&wt, &before).await {
5749                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5750                        return Err(e);
5751                    }
5752                    return self
5753                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5754                        .await;
5755                }
5756                rec.repair = Repair::Done {
5757                    before: before.clone(),
5758                    after: before,
5759                    committed: false,
5760                    failed: Some("interrupted; rolled back".to_owned()),
5761                };
5762                self.state
5763                    .event("recovery", "the repair was interrupted and rolled back");
5764                self.state.recovery = Some(rec);
5765                self.state.save()?;
5766                return Ok(());
5767            }
5768            Repair::NotStarted => {}
5769        }
5770        if crate::recovery::recheck(&mut rec, &head) {
5771            self.state.event("recovery", rec.summary());
5772            self.state.recovery = Some(rec);
5773            self.state.save()?;
5774            return Ok(());
5775        }
5776        rec.repair = Repair::Started {
5777            before: head.clone(),
5778        };
5779        self.state.recovery = Some(rec.clone());
5780        self.state.save()?;
5781
5782        let language = self.state.config.graph.language.clone();
5783        let sessions = self.state.config.graph.sessions;
5784        let artifacts = agent::artifacts_dir(&self.state.dir());
5785        let fix_prompt = prompt::recovery_fix(
5786            &self.state.instruction,
5787            &rec,
5788            self.state.config.graph.review_rounds,
5789            &language,
5790        );
5791        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5792        self.state.event(
5793            "recovery",
5794            format!(
5795                "repair: the fixer starts from the failed verification on {}",
5796                short(&head)
5797            ),
5798        );
5799        let (_job, seat, out) = self
5800            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5801                prompt: fix_prompt.clone(),
5802                spec,
5803                seat,
5804                cwd: wt.clone(),
5805                timeout,
5806                allow_write: true,
5807                sessions,
5808                artifacts: artifacts.clone(),
5809                stem: "fix-recovery".to_owned(),
5810                handover: None,
5811            })
5812            .await;
5813        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5814            rec.repair = Repair::WaitingForRestore { before: head };
5815            self.state.recovery = Some(rec);
5816            return self.stop_for_resource();
5817        }
5818        let failed = match &out {
5819            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5820            AgentOutcome::Quota(o) => {
5821                self.state.quota.push(QuotaLoss {
5822                    seat: seat.key.clone(),
5823                    node: "fix".to_owned(),
5824                    at: Timestamp::now(),
5825                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5826                });
5827                Some("rate limited (quota); the fixer could not run".to_owned())
5828            }
5829            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5830            AgentOutcome::Failed(e) => Some(e.clone()),
5831        };
5832        self.state.seats.insert(seat.key.clone(), seat);
5833        let coordination = self.coordinate(&wt).await?;
5834        let failed = match failed {
5835            // The agent's partial edits are not kept.
5836            Some(why) => {
5837                if let Err(e) = git::restore_to(&wt, &head).await {
5838                    return self
5839                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5840                        .await;
5841                }
5842                Some(why)
5843            }
5844            None => {
5845                if let Ok(r) = git::rescue_commit(
5846                    &wt,
5847                    "magi: recovery repair of the failed verification (uncommitted work)",
5848                )
5849                .await
5850                {
5851                    self.state.note_withheld("fix", &r.withheld);
5852                }
5853                None
5854            }
5855        };
5856        let after = git::rev_parse(&wt, "HEAD").await?;
5857        drop(coordination);
5858        self.state.event(
5859            "recovery",
5860            match &failed {
5861                Some(why) => format!("repair: nothing kept ({why})"),
5862                None if after != head => {
5863                    format!("repair: committed {}", short(&after))
5864                }
5865                None => "repair: the fixer made no commit".to_owned(),
5866            },
5867        );
5868        rec.repair = Repair::Done {
5869            before: head.clone(),
5870            committed: after != head,
5871            after,
5872            failed,
5873        };
5874        self.state.recovery = Some(rec);
5875        self.state.save()?;
5876        Ok(())
5877    }
5878
5879    /// A repair whose rollback could not be confirmed: the run is blocked and
5880    /// resumable, and the unconfirmed state is never reviewed.
5881    async fn stop_recovery(
5882        &mut self,
5883        mut rec: crate::recovery::RecoveryRecord,
5884        head: &str,
5885        why: String,
5886    ) -> Result<()> {
5887        // `Started` stays recorded: a resume retries the restore.
5888        rec.repair = crate::recovery::Repair::Started {
5889            before: head.to_owned(),
5890        };
5891        self.state.recovery = Some(rec);
5892        self.state
5893            .event("recovery", format!("repair could not be undone: {why}"));
5894        self.state.status = RunStatus::Blocked;
5895        self.state.save()?;
5896        bail!("recovery repair could not be rolled back: {why}")
5897    }
5898
5899    async fn review_loop(&mut self) -> Result<()> {
5900        loop {
5901            if self
5902                .state
5903                .base_sync
5904                .as_ref()
5905                .is_some_and(|s| s.conflict.is_some())
5906            {
5907                return Ok(());
5908            }
5909            if self.catchup_repairable() {
5910                let record = self.state.reviews.last().expect("repairable round").clone();
5911                let winner = self
5912                    .state
5913                    .winner()
5914                    .cloned()
5915                    .context("catch-up repair has no winner")?;
5916                if !self.fix_review_round(&winner, record).await? {
5917                    return Ok(());
5918                }
5919            }
5920            self.review_loop_pass().await?;
5921            if !self.catchup_repairable() {
5922                return Ok(());
5923            }
5924        }
5925    }
5926
5927    /// Derive an owed catch-up repair from durable round evidence, including
5928    /// reentry after a crash before the fixer was dispatched.
5929    fn catchup_repairable(&self) -> bool {
5930        self.state.reviews.len() < self.state.config.graph.review_rounds
5931            && self.state.reviews.last().is_some_and(|r| {
5932                !r.clean
5933                    && r.blocking == 0
5934                    && r.fix.is_none()
5935                    && r.e2e_status() == E2eStatus::Failed
5936            })
5937    }
5938
5939    async fn review_loop_pass(&mut self) -> Result<()> {
5940        // A base that would not rebase is a person's decision, not a review
5941        // round: nothing here would change the answer, and reviewers and a
5942        // fixer would be spending real budget on a tree that cannot land
5943        // regardless of what they find.
5944        if self
5945            .state
5946            .base_sync
5947            .as_ref()
5948            .is_some_and(|s| s.conflict.is_some())
5949        {
5950            return Ok(());
5951        }
5952        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5953        // agent files with `magi task add` name the run that paid for it. The
5954        // prompt overlay is cloned alongside it because the waves borrow it
5955        // while `self` is mutably borrowed by the node's own bookkeeping.
5956        let run_id = self.state.id.clone();
5957        let prompts = self.state.config.prompts.clone();
5958        let Some(winner) = self.state.winner().cloned() else {
5959            return Ok(());
5960        };
5961        let max_rounds = self.state.config.graph.review_rounds;
5962        // A clean round, an exhausted round budget, or a stalled tree (see
5963        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5964        // they are recorded — recomputed here, not read off `status`, so a
5965        // reentry into a run that already stopped restates the identical
5966        // verdict instead of silently handing back whatever an earlier node
5967        // in this same walk clobbered `status` to (a solo-candidate
5968        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5969        // below runs an empty range once the budget is spent, and would
5970        // otherwise fall through without touching `status` at all.
5971        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5972            // A reentry after a crash between the last round's save and
5973            // `stop_reviewing` reaches the hand-off here, not there.
5974            if status == RunStatus::Gating {
5975                self.record_contested_handoff();
5976            }
5977            self.state.status = status;
5978            self.state.save()?;
5979            return Ok(());
5980        }
5981        self.state.status = RunStatus::Reviewing;
5982        // A last recorded round whose own verification never resolved
5983        // (`ResourceBlocked` — the shared build cache, not the patch) is
5984        // never a concluded round, whatever the round budget says: starting
5985        // a fresh round on top of it would spend a whole new reviewer wave
5986        // re-reading an unchanged patch instead of just retrying the one
5987        // check that actually needs it, and once the budget is spent the
5988        // loop below has nothing left to do at all (its range is empty).
5989        // Retry that check directly instead, exactly the same retry
5990        // `stop_reviewing` already does for its own catch-up case.
5991        if self.state.reviews.last().is_some_and(|r| {
5992            r.e2e_status() == E2eStatus::ResourceBlocked
5993                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5994        }) {
5995            let shell = self.state.config.shell();
5996            return self
5997                .stop_unresolved_review(
5998                    "the last round's own verification never resolved",
5999                    &shell,
6000                    &winner.worktree,
6001                )
6002                .await;
6003        }
6004
6005        self.recovery_repair(&winner).await?;
6006        if self.resource_halt()? {
6007            return Ok(());
6008        }
6009
6010        let repo = self.state.repo.clone();
6011        let root = self.state.worktree_root();
6012        let language = self.state.config.graph.language.clone();
6013        let sessions = self.state.config.graph.sessions;
6014        let artifacts = self.merge_repair_artifacts();
6015        let base = self.landing_base();
6016        let base_short = short(&base);
6017        let reviewers = self.roles.reviewers.clone();
6018        let shell = self.state.config.shell();
6019
6020        for round in (self.state.reviews.len() + 1)..=max_rounds {
6021            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6022            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6023            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6024            // The prior round's own record, already persisted — never a
6025            // hand-carried variable of just its failing output: that is
6026            // exactly what let a round's e2e result drift out of sync with
6027            // which commit it was actually about (see `SCHEMA`'s doc for
6028            // schema 8). Judged against `head`, the commit reviewers are
6029            // about to look at now, so the summary always reads as "an
6030            // earlier head" here — this round's own patch has not been
6031            // checked yet.
6032            let prev_verification = self
6033                .state
6034                .reviews
6035                .last()
6036                .and_then(|r| r.verification_summary(&head));
6037            // Only the first round of a run that took a branch over: later
6038            // rounds have this run's own history.
6039            let prior_section = match (&self.state.recovery, round) {
6040                (Some(rec), 1) if self.state.reviews.is_empty() => {
6041                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6042                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6043                            Some(crate::recovery::head_of(
6044                                &git::diff(&winner.worktree, reviewed, "HEAD")
6045                                    .await
6046                                    .unwrap_or_default(),
6047                                crate::recovery::SINCE_DIFF_MAX,
6048                            ))
6049                        }
6050                        _ => None,
6051                    };
6052                    Some(prompt::recovery_review(rec, since.as_deref()))
6053                }
6054                _ => None,
6055            };
6056
6057            // Each reviewer gets its own detached checkout of exactly this
6058            // commit: nobody can perturb the winner's tree, and the fixer can
6059            // keep working without racing a reviewer.
6060            let mut jobs = Vec::new();
6061            let coordination = self.coordinate(&repo).await?;
6062            for (r, spec) in reviewers.iter().cloned().enumerate() {
6063                let wt = root.join(format!("review-{}", r + 1));
6064                // `clean -fdx` in a reset wipes setup's products, so every
6065                // round sets the seat up again.
6066                if wt.exists() {
6067                    git::reset_detached(&wt, &head).await?;
6068                } else {
6069                    git::worktree_add_detached(&repo, &wt, &head).await?;
6070                }
6071                self.setup_seat_worktree(&repo, &wt, false).await?;
6072                let seat_key = format!("review-{}", r + 1);
6073                // The seat starts the round on whoever answered it last, not
6074                // on the agent the spec names, so a failure is not re-paid.
6075                let spec = pick_start_spec(
6076                    &self.roles.reviewer_roster,
6077                    spec,
6078                    self.state.seat_history.get(&seat_key),
6079                );
6080                let seat = self.seat(&seat_key, &spec.id);
6081                jobs.push(SeatJob {
6082                    prompt: prompt::review(&prompt::ReviewCtx {
6083                        instruction: &self.state.instruction,
6084                        branch: &winner.branch,
6085                        base_short: &base_short,
6086                        stat: &stat,
6087                        patch: &patch,
6088                        verification: prev_verification.as_ref(),
6089                        prior: prior_section.as_deref(),
6090                        reviewers: reviewers.len(),
6091                        round,
6092                        rounds: max_rounds,
6093                        // A review-only run has no rankings, so nothing
6094                        // competed for this patch and the reviewer is told so.
6095                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6096                        lens: Lens::for_seat(r),
6097                        language: &language,
6098                        github_language: &self.state.config.graph.github_language,
6099                    }),
6100                    spec,
6101                    seat,
6102                    cwd: wt,
6103                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6104                    allow_write: false,
6105                    sessions,
6106                    artifacts: artifacts.clone(),
6107                    stem: format!("review-{round}-{}", r + 1),
6108                    handover: None,
6109                });
6110            }
6111
6112            drop(coordination);
6113            self.state.event(
6114                "review",
6115                format!(
6116                    "round {round}: {} reviewers on {}",
6117                    jobs.len(),
6118                    short(&head)
6119                ),
6120            );
6121            let mut quota_losses = Vec::new();
6122            let review_retries = self.state.config.graph.retries;
6123            let review_cache = self.state.config.cache_dir();
6124            let ctx = WaveCtx {
6125                carry_seats: true,
6126                pauses: Some((&self.pause, &self.interrupt)),
6127                run: &run_id,
6128                node: "review",
6129                prompts: &prompts,
6130                cache: review_cache.as_deref(),
6131                round: Some(round),
6132            };
6133            let results = ask_json_wave::<Review>(
6134                jobs,
6135                Arc::clone(&self.sem),
6136                review_retries,
6137                &self.roles.reviewer_roster,
6138                &ctx,
6139                &mut quota_losses,
6140                &mut self.state,
6141                &|_: &Review| Ok(()),
6142            )
6143            .await;
6144            if self.state.resource_block.is_some() {
6145                return self.stop_for_resource();
6146            }
6147            // Counted before the move below: how many of *this* round's
6148            // reviewer seats were lost to their own rate limit, as opposed to
6149            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6150            let round_quota_missing = quota_losses.len();
6151            self.state.quota.extend(quota_losses);
6152
6153            let mut records = Vec::new();
6154            let mut all_findings = Vec::new();
6155            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6156                let agent_id = seat.agent.clone();
6157                self.state.seats.insert(seat.key.clone(), seat);
6158                let mut record = ReviewRecord {
6159                    reviewer: r + 1,
6160                    agent: agent_id,
6161                    summary: String::new(),
6162                    findings: Vec::new(),
6163                    vote: None,
6164                    failed: None,
6165                    duration_ms: 0,
6166                    // Set for both outcomes: `failed: Some(_)` with
6167                    // `attempts > 0` is a seat every retry still lost, not a
6168                    // recovered one — only `failed: None` with `attempts > 0`
6169                    // reads as "answered after a nudge" (see this field's own
6170                    // doc).
6171                    attempts,
6172                };
6173                match res {
6174                    Ok((review, out)) => {
6175                        // Sanitized here, at the point every other piece of
6176                        // agent prose in this file is (candidate summaries,
6177                        // deliberation turns, vote reasons): a reviewer's own
6178                        // words are the one thing about it that could name
6179                        // it, and reconsideration below broadcasts this same
6180                        // summary and these same findings to every other
6181                        // seat on the panel.
6182                        record.summary =
6183                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6184                        record.vote = Some(review.vote);
6185                        record.duration_ms = out.duration_ms;
6186                        for (n, mut f) in review.findings.into_iter().enumerate() {
6187                            // ids are magi's, never the agent's: the fixer's
6188                            // adoption report is keyed by them.
6189                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6190                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6191                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6192                            // `file` is agent-supplied prose too, never
6193                            // checked against the real tree — the same
6194                            // exposure `title`/`detail` above have, just in
6195                            // a field easy to forget because it looks like a
6196                            // path rather than free text.
6197                            f.file = f
6198                                .file
6199                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6200                            all_findings.push(f.clone());
6201                            record.findings.push(f);
6202                        }
6203                        self.state.event(
6204                            "review",
6205                            format!(
6206                                "round {round}: reviewer {} voted {} with {} finding(s)",
6207                                r + 1,
6208                                review.vote.label(),
6209                                record.findings.len()
6210                            ),
6211                        );
6212                    }
6213                    Err(e) => {
6214                        record.failed = Some(e.to_string());
6215                        self.state.event(
6216                            "review",
6217                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6218                        );
6219                    }
6220                }
6221                records.push(record);
6222            }
6223
6224            // Tally the round's votes and, if they split, spend the one
6225            // round of reconsideration the split -> deliberate -> revote
6226            // shape `judge`/`vote` use for the panel, sized down to what a
6227            // read-only review round can afford: one round, and a revote
6228            // rather than an argument, because the panel already wrote its
6229            // reasoning down as findings the first time around.
6230            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6231            let vote_split =
6232                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6233            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6234            if vote_split {
6235                self.state.event(
6236                    "review",
6237                    format!(
6238                        "round {round}: votes split ({}) — one round of reconsideration",
6239                        initial_votes
6240                            .iter()
6241                            .map(|v| v.label())
6242                            .collect::<Vec<_>>()
6243                            .join(", ")
6244                    ),
6245                );
6246                // Seats read every seat's findings and votes, still numbered
6247                // and never named — the same anonymity `review` itself keeps.
6248                let panel: Vec<ReviewSeatReport<'_>> = records
6249                    .iter()
6250                    .filter_map(|r| {
6251                        r.vote.map(|vote| ReviewSeatReport {
6252                            reviewer: r.reviewer,
6253                            vote,
6254                            summary: &r.summary,
6255                            findings: &r.findings,
6256                        })
6257                    })
6258                    .collect();
6259
6260                let mut jobs = Vec::new();
6261                let mut seats_at = Vec::new();
6262                for (r, spec) in reviewers.iter().cloned().enumerate() {
6263                    // A seat with no initial vote has nothing to reconsider
6264                    // from and stays absent, the same as it stayed absent
6265                    // from `panel` above.
6266                    if records[r].vote.is_none() {
6267                        continue;
6268                    }
6269                    let wt = root.join(format!("review-{}", r + 1));
6270                    let seat_key = format!("review-{}", r + 1);
6271                    let spec = self.occupant(&seat_key, spec);
6272                    let seat = self.seat(&seat_key, &spec.id);
6273                    // A seat with no live session has already forgotten the
6274                    // initial review's prompt — restate the patch it is
6275                    // voting on, the same as `deliberate`/`vote` do for a
6276                    // judge in the same position.
6277                    // The panel already carries this seat's own review and
6278                    // vote, so restating the patch makes the prompt whole for
6279                    // a seat handed to another agent.
6280                    let build = |with_patch: bool| {
6281                        prompt::review_reconsider(&ReviewReconsiderCtx {
6282                            instruction: &self.state.instruction,
6283                            reviewer: r + 1,
6284                            lens: Lens::for_seat(r),
6285                            panel: &panel,
6286                            patch: with_patch.then_some(ReviewPatch {
6287                                branch: &winner.branch,
6288                                base_short: &base_short,
6289                                stat: &stat,
6290                                patch: &patch,
6291                            }),
6292                            round,
6293                            rounds: max_rounds,
6294                            language: &language,
6295                        })
6296                    };
6297                    let full = build(true);
6298                    let prompt = if has_context(&spec, &seat, sessions) {
6299                        build(false)
6300                    } else {
6301                        full.clone()
6302                    };
6303                    jobs.push(SeatJob {
6304                        prompt,
6305                        spec,
6306                        seat,
6307                        cwd: wt,
6308                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6309                        allow_write: false,
6310                        sessions,
6311                        artifacts: artifacts.clone(),
6312                        stem: format!("review-{round}-reconsider-{}", r + 1),
6313                        handover: Some(full),
6314                    });
6315                    seats_at.push(r);
6316                }
6317
6318                let mut recon_quota_losses = Vec::new();
6319                let recon_cache = self.state.config.cache_dir();
6320                let recon_ctx = WaveCtx {
6321                    carry_seats: true,
6322                    pauses: Some((&self.pause, &self.interrupt)),
6323                    run: &run_id,
6324                    node: "review",
6325                    prompts: &prompts,
6326                    cache: recon_cache.as_deref(),
6327                    round: Some(round),
6328                };
6329                let recon_results = ask_json_wave::<ReviewRevote>(
6330                    jobs,
6331                    Arc::clone(&self.sem),
6332                    review_retries,
6333                    &self.roles.reviewer_roster,
6334                    &recon_ctx,
6335                    &mut recon_quota_losses,
6336                    &mut self.state,
6337                    &|_: &ReviewRevote| Ok(()),
6338                )
6339                .await;
6340                if self.state.resource_block.is_some() {
6341                    return self.stop_for_resource();
6342                }
6343                self.state.quota.extend(recon_quota_losses);
6344
6345                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6346                    let agent_id = seat.agent.clone();
6347                    self.state.seats.insert(seat.key.clone(), seat);
6348                    let mut rec = ReviewRevoteRecord {
6349                        reviewer: r + 1,
6350                        agent: agent_id,
6351                        vote: None,
6352                        reason: String::new(),
6353                        failed: None,
6354                    };
6355                    match res {
6356                        Ok((rv, _)) => {
6357                            rec.vote = Some(rv.vote);
6358                            rec.reason =
6359                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6360                            self.state.event(
6361                                "review",
6362                                format!(
6363                                    "round {round}: reviewer {} revoted {}",
6364                                    r + 1,
6365                                    rv.vote.label()
6366                                ),
6367                            );
6368                        }
6369                        Err(e) => {
6370                            rec.failed = Some(e.to_string());
6371                            self.state.event(
6372                                "review",
6373                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6374                            );
6375                        }
6376                    }
6377                    reconsideration.push(rec);
6378                }
6379            } else if initial_votes.len() > 1 {
6380                self.state.event(
6381                    "review",
6382                    format!(
6383                        "round {round}: votes agreed ({}) — no reconsideration",
6384                        initial_votes[0].label()
6385                    ),
6386                );
6387            }
6388
6389            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6390            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6391            // A round that already has a blocking finding and a round left to
6392            // try is going back to the fixer no matter what `verify.e2e`
6393            // says, so running it first only spends the loop's slowest step
6394            // (minutes, for a Rust repo's full test suite) on a head about
6395            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6396            // runs once a round has no blocking findings left (see
6397            // `round_is_clean`, which a deferred — empty — `e2e` can never
6398            // satisfy since `blocking` is nonzero whenever this branch is
6399            // taken), and `stop_reviewing` forces a real run before it will
6400            // ever read a deferred round as green.
6401            let defer_e2e =
6402                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6403            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6404                let reason =
6405                    format!("{blocking} blocking finding(s) already required a fix this round");
6406                self.state.event(
6407                    "verify",
6408                    format!(
6409                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6410                         {}); it will run once a round has none left",
6411                        short(&head)
6412                    ),
6413                );
6414                (Vec::new(), false, true, Some(reason))
6415            } else {
6416                let e2e_commands = self.state.config.verify.e2e.clone();
6417                let cache_dir = self.state.config.cache_dir();
6418                let context = format!("round {round}");
6419                let wait = self.repository_wait();
6420                let (e2e, verify_retried) = with_cache_lease(
6421                    &mut self.state,
6422                    cache_dir.as_deref(),
6423                    "e2e",
6424                    "e2e",
6425                    &winner.worktree,
6426                    &head,
6427                    verify_timeout,
6428                    &context,
6429                    &wait,
6430                    |state, budget| {
6431                        let shell = shell.clone();
6432                        let e2e_commands = e2e_commands.clone();
6433                        let worktree = winner.worktree.clone();
6434                        let context = context.clone();
6435                        async move {
6436                            run_e2e_with_retry(
6437                                state,
6438                                &shell,
6439                                &e2e_commands,
6440                                &worktree,
6441                                budget,
6442                                &context,
6443                            )
6444                            .await
6445                        }
6446                    },
6447                )
6448                .await?;
6449                (e2e, verify_retried, false, None)
6450            };
6451
6452            let expected = records.len();
6453            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6454            let incomplete = answered < expected;
6455            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6456            let policy = self.state.config.graph.incomplete_review;
6457            let clean = round_is_clean(
6458                blocking,
6459                e2e_ok,
6460                answered,
6461                expected,
6462                round_quota_missing,
6463                policy,
6464            );
6465
6466            let mut round_record = ReviewRound {
6467                round,
6468                head: head.clone(),
6469                verified_head: None,
6470                verified_at: None,
6471                reviews: records,
6472                e2e,
6473                verify_retried,
6474                e2e_deferred,
6475                e2e_defer_reason,
6476                fix: None,
6477                blocking,
6478                answered,
6479                expected,
6480                clean,
6481                progressed: false,
6482                vote_split,
6483                reconsideration,
6484                verdict: None,
6485            };
6486            // The final vote per seat is its revote where reconsideration
6487            // ran and answered, its initial vote otherwise — the same
6488            // fallback `tally` uses for a judge whose private vote failed.
6489            round_record.verdict = ReviewVote::worst(
6490                round_record
6491                    .final_votes()
6492                    .into_iter()
6493                    .map(|(_, _, vote)| vote),
6494            );
6495            // Which commit and when magi actually attempted to check —
6496            // known the moment a command was dispatched against `head`,
6497            // whether or not it finished: a resource-blocked attempt still
6498            // targeted a specific commit at a specific time, and leaving
6499            // that unrecorded is exactly what made `verification_summary`
6500            // report a fresh attempt as "commit unknown ... recorded before
6501            // this was tracked", indistinguishable from a genuinely old,
6502            // untracked record. Only a deferred or unconfigured round never
6503            // ran at all and has nothing to record — see
6504            // `ReviewRound::verified_head`'s own doc.
6505            if !matches!(
6506                round_record.e2e_status(),
6507                E2eStatus::Deferred | E2eStatus::NotConfigured
6508            ) {
6509                round_record.verified_head = Some(head.clone());
6510                round_record.verified_at = Some(Timestamp::now());
6511            }
6512
6513            if incomplete {
6514                let missing: Vec<String> = round_record
6515                    .reviews
6516                    .iter()
6517                    .filter(|r| r.failed.is_some())
6518                    .map(|r| format!("review-{}", r.reviewer))
6519                    .collect();
6520                self.state.event(
6521                    "review",
6522                    format!(
6523                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6524                        missing.join(", ")
6525                    ),
6526                );
6527            }
6528
6529            if clean {
6530                self.state.event(
6531                    "review",
6532                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6533                        format!(
6534                            "round {round}: clean (warn policy, incomplete panel) — no \
6535                             blocking findings from the seats that answered, verification green"
6536                        )
6537                    } else if incomplete {
6538                        format!(
6539                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6540                             quorum) — no blocking findings from the seats that answered, \
6541                             verification green",
6542                            expected - answered
6543                        )
6544                    } else {
6545                        format!("round {round}: clean — no blocking findings, verification green")
6546                    },
6547                );
6548                self.state.reviews.push(round_record);
6549                self.state.status = RunStatus::Gating;
6550                self.state.save()?;
6551                return Ok(());
6552            }
6553
6554            // Nothing was raised and verification passed, but not every seat
6555            // answered and `round_is_clean` still refused to call it clean —
6556            // either a seat is missing for a reason other than its own quota
6557            // (a crash, a timeout, unparsable output — worth another try), or
6558            // every seat that could have answered lost its quota and nobody
6559            // is left to decide on: re-review rather than send the fixer
6560            // after a round with nothing to fix.
6561            if incomplete && blocking == 0 && e2e_ok {
6562                self.state.reviews.push(round_record);
6563                self.state.save()?;
6564                if round == max_rounds {
6565                    self.state.status = RunStatus::Blocked;
6566                    self.state.event(
6567                        "review",
6568                        format!(
6569                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6570                             refusing to call it clean",
6571                            expected - answered
6572                        ),
6573                    );
6574                    return Ok(());
6575                }
6576                continue;
6577            }
6578
6579            // Nothing for the fixer to act on (`blocking == 0`) and the only
6580            // reason this round is not clean is that magi itself never got
6581            // a command to run — the shared build cache, not the patch (see
6582            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6583            // the fixer would invite a change to appease contention that has
6584            // nothing to do with the diff, and would leave this attempt
6585            // sitting in the next round's prompt as if it were about an
6586            // earlier, superseded commit rather than what it actually is:
6587            // the same head, still waiting to be checked. Wait for it the
6588            // same way the final round's own contention is already handled,
6589            // whatever round this happens to be.
6590            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6591                self.state.reviews.push(round_record);
6592                return self
6593                    .stop_unresolved_review(
6594                        "the round's own verification could not run",
6595                        &shell,
6596                        &winner.worktree,
6597                    )
6598                    .await;
6599            }
6600
6601            if round == max_rounds {
6602                self.state.reviews.push(round_record);
6603                return self
6604                    .stop_reviewing(
6605                        &format!(
6606                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6607                        ),
6608                        &shell,
6609                        &winner.worktree,
6610                    )
6611                    .await;
6612            }
6613
6614            if !self.fix_review_round(&winner, round_record).await? {
6615                return Ok(());
6616            }
6617        }
6618        Ok(())
6619    }
6620
6621    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6622    /// False means admission or continuation policy stopped the review loop.
6623    async fn fix_review_round(
6624        &mut self,
6625        winner: &Candidate,
6626        mut round_record: ReviewRound,
6627    ) -> Result<bool> {
6628        let round = round_record.round;
6629        let shell = self.state.config.shell();
6630        let step = self.run_fix_round(winner, &mut round_record).await?;
6631        if matches!(step, FixStep::Resource) {
6632            self.stop_for_resource()?;
6633            return Ok(false);
6634        }
6635        if self.state.reviews.len() < round {
6636            self.state.reviews.push(round_record);
6637        } else {
6638            self.state.reviews[round - 1] = round_record;
6639        }
6640        self.state.save()?;
6641
6642        // The fixer's own report never came back this round, even after
6643        // `continue_fix_report`'s own budget was spent on it — not an
6644        // ordinary "no report" (dropped stream, quota, plain failure),
6645        // which already reads that way and is left to the existing round
6646        // budget. Stopping here, rather than opening another round, is
6647        // what keeps a next reviewer/fixer wave from ever being
6648        // dispatched onto `winner.worktree` while whatever the seat's
6649        // last call may still have running there is unaccounted for: no
6650        // process liveness check exists (and none is being added — see
6651        // AGENTS.md/this task's own scope), so the only way to honour
6652        // "nothing starts before a valid report returns" is to not start
6653        // anything further on this worktree from this run at all.
6654        if matches!(step, FixStep::Recorded { report_lost: true }) {
6655            self.stop_reviewing(
6656                "the fixer's adoption report never came back, even after resuming its \
6657                         own seat; refusing to start another round against the same worktree \
6658                         while that is unresolved",
6659                &shell,
6660                &winner.worktree,
6661            )
6662            .await?;
6663            return Ok(false);
6664        }
6665
6666        let streak = self
6667            .state
6668            .reviews
6669            .iter()
6670            .rev()
6671            .take_while(|r| !r.progressed)
6672            .count();
6673        if streak >= STAGNANT_LIMIT {
6674            self.stop_reviewing(
6675                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6676                &shell,
6677                &winner.worktree,
6678            )
6679            .await?;
6680            return Ok(false);
6681        }
6682        Ok(true)
6683    }
6684
6685    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6686    async fn run_fix_round(
6687        &mut self,
6688        winner: &Candidate,
6689        round_record: &mut ReviewRound,
6690    ) -> Result<FixStep> {
6691        let round = round_record.round;
6692        let max_rounds = self.state.config.graph.review_rounds;
6693        let run_id = self.state.id.clone();
6694        let prompts = self.state.config.prompts.clone();
6695        let language = self.state.config.graph.language.clone();
6696        let sessions = self.state.config.graph.sessions;
6697        let artifacts = self.merge_repair_artifacts();
6698        let base = self.landing_base();
6699        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6700        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6701        let this_round_verification = round_record.verification_summary(&head);
6702        let all_findings: Vec<_> = round_record
6703            .reviews
6704            .iter()
6705            .flat_map(|r| r.findings.iter().cloned())
6706            .collect();
6707        // Fix. The winner's own implementer seat continues its conversation:
6708        // the competition is over, so context is pure benefit now.
6709        let blocking_findings: Vec<_> = all_findings
6710            .iter()
6711            .filter(|f| f.severity.blocks())
6712            .cloned()
6713            .collect();
6714        let fix_prompt = prompt::fix_in(
6715            &self.state.instruction,
6716            &blocking_findings,
6717            this_round_verification.as_ref(),
6718            round,
6719            max_rounds,
6720            &language,
6721            &self.state.config.graph.github_language,
6722        );
6723        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6724        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6725        let (job, seat, out) = self
6726            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6727                prompt: fix_prompt.clone(),
6728                spec,
6729                seat,
6730                cwd: winner.worktree.clone(),
6731                timeout,
6732                allow_write: true,
6733                sessions,
6734                artifacts: artifacts.clone(),
6735                stem: format!("fix-{round}"),
6736                handover: None,
6737            })
6738            .await;
6739        // Not asked: the round is not recorded (so a resume repeats it),
6740        // no fixer was handed over to, and the seat is left as it was.
6741        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6742            return Ok(FixStep::Resource);
6743        }
6744        let agent_id = seat.agent.clone();
6745
6746        let mut fix = FixRecord {
6747            agent: agent_id,
6748            addressed: Vec::new(),
6749            rejected: Vec::new(),
6750            notes: String::new(),
6751            committed: false,
6752            failed: None,
6753            duration_ms: 0,
6754            continuation: None,
6755        };
6756        let mut continuation = ContinuationRecord::not_needed();
6757        let mut final_seat = seat.clone();
6758        match out {
6759            AgentOutcome::Ok(o) => {
6760                fix.duration_ms = o.duration_ms;
6761                let parsed = verdict::extract_json::<FixReport>(&o.text);
6762                // A parsed report standing next to a command this same
6763                // reply's own CLI never confirmed the exit status of is
6764                // not a resolved answer — the identical `CommandEvidence`
6765                // `state.jobs` renders, read here instead of only on
6766                // display, per the completion judgment and the shown
6767                // record needing to agree.
6768                let incomplete_reason = match &parsed {
6769                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6770                        "the reply parsed, but it reported a command whose own CLI never \
6771                             confirmed an exit status"
6772                            .to_owned(),
6773                    ),
6774                    Ok(_) => None,
6775                    Err(e) => Some(e.to_string()),
6776                };
6777                match incomplete_reason {
6778                    None => {
6779                        let report = parsed.expect("checked Ok above");
6780                        fix.addressed = report.addressed;
6781                        fix.rejected = report.rejected;
6782                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6783                    }
6784                    Some(reason) => {
6785                        let (resumed_seat, resolved, failure, cont) = self
6786                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6787                            .await;
6788                        fix.duration_ms += cont.cumulative_wait_ms;
6789                        continuation = cont;
6790                        final_seat = resumed_seat;
6791                        match resolved {
6792                            Some(report) => {
6793                                fix.addressed = report.addressed;
6794                                fix.rejected = report.rejected;
6795                                fix.notes =
6796                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6797                            }
6798                            None => fix.failed = failure,
6799                        }
6800                    }
6801                }
6802            }
6803            // The CLI's raw error JSON is not a fix report to parse.
6804            AgentOutcome::Dropped(o) => {
6805                fix.duration_ms = o.duration_ms;
6806                let why = o
6807                    .dropped
6808                    .as_ref()
6809                    .map(|d| d.why.as_str())
6810                    .unwrap_or("the CLI ended the stream without delivering its answer");
6811                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6812            }
6813            AgentOutcome::Quota(o) => {
6814                self.state.quota.push(QuotaLoss {
6815                    seat: final_seat.key.clone(),
6816                    node: "fix".to_owned(),
6817                    at: Timestamp::now(),
6818                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6819                });
6820                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6821            }
6822            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6823        }
6824        fix.continuation = Some(continuation);
6825        self.state.seats.insert(final_seat.key.clone(), final_seat);
6826        let coordination = self.coordinate(&winner.worktree).await?;
6827        if let Ok(r) = git::rescue_commit(
6828            &winner.worktree,
6829            &format!("magi: review round {round} fixes (uncommitted work)"),
6830        )
6831        .await
6832        {
6833            self.state.note_withheld("fix", &r.withheld);
6834        }
6835        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6836        fix.committed = after != before;
6837        // Judged by what `git` says moved against base, never by the
6838        // fixer's own `addressed`/`rejected` count — see
6839        // `ReviewRound::progressed`. Propagated with `?`, the same as the
6840        // `patch` snapshot above: swallowing this error would default
6841        // `diff_after` to empty, which almost always differs from a
6842        // non-empty `patch` and reads as "progressed" — exactly backwards
6843        // for a `git` failure the stagnation check cannot see through.
6844        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6845        drop(coordination);
6846        let progressed = diff_after != patch;
6847        let commit_note = if fix.committed {
6848            "committed"
6849        } else {
6850            "NO new commit"
6851        };
6852        let tree_note = if progressed {
6853            "changed vs base"
6854        } else {
6855            "unchanged vs base"
6856        };
6857        self.state.event(
6858            "fix",
6859            match &fix.failed {
6860                // Distinct on purpose from "0 addressed, 0 rejected": the
6861                // fixer's own diff still landed (blocking counts do keep
6862                // falling round over round), only its adoption report did
6863                // not come back, so this must never read like every
6864                // finding was reviewed and declined.
6865                Some(reason) => {
6866                    format!(
6867                        "round {round}: fixer's adoption report was lost ({reason}); \
6868                             {commit_note}, tree {tree_note}"
6869                    )
6870                }
6871                None => format!(
6872                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6873                         {tree_note}{}",
6874                    fix.addressed.len(),
6875                    fix.rejected.len(),
6876                    if continuation.outcome == ContinuationOutcome::Resumed {
6877                        format!(
6878                            " (adoption report recovered after {} continuation(s))",
6879                            continuation.attempts
6880                        )
6881                    } else {
6882                        String::new()
6883                    },
6884                ),
6885            },
6886        );
6887        round_record.fix = Some(fix);
6888        round_record.progressed = progressed;
6889        Ok(FixStep::Recorded {
6890            report_lost: matches!(
6891                continuation.outcome,
6892                ContinuationOutcome::Exhausted
6893                    | ContinuationOutcome::QuotaLost
6894                    | ContinuationOutcome::NoSession
6895            ),
6896        })
6897    }
6898
6899    /// Decide, from the last recorded round's own verification, whether
6900    /// stopping the review loop is a hand-off or a genuine block.
6901    ///
6902    /// Called once the loop has given up trying — the round budget is spent,
6903    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6904    /// findings still open, never while a round is still clean or the
6905    /// incomplete-panel case handled inline above. Gate and e2e are facts
6906    /// about the tree; a lingering review finding is an opinion, and this
6907    /// workload's own `magi stats` puts reviewer precision low enough
6908    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6909    /// must not by itself stand between a green, verified change and the
6910    /// human who decides what to do with it. A red e2e is not an opinion, so
6911    /// that case still blocks, with the failing command and a tail of its
6912    /// output recorded here rather than left in `run.json` for someone to go
6913    /// find.
6914    ///
6915    /// A round that deferred its own e2e (see [`Config::graph`]'s
6916    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6917    /// only because nothing ran, and treating an empty list as a passing one
6918    /// here is exactly the "deferred painted green" bug this function exists
6919    /// to not have. When the last round's own verification never resolved —
6920    /// deferred on purpose, or a real attempt the shared build cache blocked
6921    /// — this makes (or retries) the real run, on the actual worktree this
6922    /// loop is about to stop touching, before deciding anything. A
6923    /// resource-blocked attempt is likewise never read as either green or
6924    /// red: it is evidence about the machine, not the patch (see
6925    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6926    /// blocked cache leaves this call without deciding rather than guessing
6927    /// — the caller retries on a later reentry.
6928    /// Record, once, that the review loop handed off over a blocking finding
6929    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6930    /// `land` asks the owner even with `land_approval` off. Called from every
6931    /// path that concludes `Gating`; a reentry keeps the first record.
6932    fn record_contested_handoff(&mut self) {
6933        if self.state.contested_handoff.is_some() {
6934            return;
6935        }
6936        let Some(contested) = self
6937            .state
6938            .reviews
6939            .last()
6940            .and_then(ReviewRound::contested_handoff)
6941        else {
6942            return;
6943        };
6944        self.state.event(
6945            "review",
6946            format!(
6947                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6948                 wait for the owner's approval",
6949                contested.findings.len(),
6950                contested.rejecters.len()
6951            ),
6952        );
6953        self.state.contested_handoff = Some(contested);
6954    }
6955
6956    async fn stop_unresolved_review(
6957        &mut self,
6958        why: &str,
6959        shell: &[String],
6960        worktree: &Path,
6961    ) -> Result<()> {
6962        self.stop_reviewing_inner(why, shell, worktree, true).await
6963    }
6964
6965    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6966        self.stop_reviewing_inner(why, shell, worktree, false).await
6967    }
6968
6969    async fn stop_reviewing_inner(
6970        &mut self,
6971        why: &str,
6972        shell: &[String],
6973        worktree: &Path,
6974        repairable: bool,
6975    ) -> Result<()> {
6976        let round_idx = self.state.reviews.len() - 1;
6977        // A deferred round and a resource-blocked one are the same shape
6978        // here: neither has a real result yet, and both get one more
6979        // attempt. Read off `e2e_status` — the single source for this —
6980        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6981        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6982        // still retries instead of being read as a settled result the
6983        // instant it stops being empty.
6984        let needs_catchup_run = matches!(
6985            self.state.reviews[round_idx].e2e_status(),
6986            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6987        );
6988        if needs_catchup_run {
6989            let round = self.state.reviews[round_idx].round;
6990            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6991            let commands = self.state.config.verify.e2e.clone();
6992            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6993            let cache_dir = self.state.config.cache_dir();
6994            let context = format!(
6995                "round {round}: verification unresolved, catching up before the final decision"
6996            );
6997            let wait = self.repository_wait();
6998            let (outcomes, verify_retried) = with_cache_lease(
6999                &mut self.state,
7000                cache_dir.as_deref(),
7001                "e2e",
7002                "e2e",
7003                worktree,
7004                &attempted_head,
7005                timeout,
7006                &context,
7007                &wait,
7008                |state, budget| {
7009                    let shell = shell.to_vec();
7010                    let commands = commands.clone();
7011                    let context = context.clone();
7012                    async move {
7013                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7014                            .await
7015                    }
7016                },
7017            )
7018            .await?;
7019            let last = &mut self.state.reviews[round_idx];
7020            last.e2e = outcomes;
7021            last.verify_retried = verify_retried;
7022            // Always the commit and time this attempt actually targeted,
7023            // whether or not it happens to equal the reviewed `head` and
7024            // whether or not a command finished — see
7025            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7026            // attempt is recorded too, so a later reader sees "attempted
7027            // again at T2" rather than silence.
7028            last.verified_head = Some(attempted_head);
7029            last.verified_at = Some(Timestamp::now());
7030            if verify_inconclusive(&last.e2e) {
7031                // Still not a real result: `e2e_deferred` is left exactly
7032                // as it was, so `needs_catchup_run` above reads
7033                // `ResourceBlocked` (via `e2e_status`, which checks
7034                // `resource_blocked` before `e2e_deferred`) and retries
7035                // again on the next reentry, rather than recording
7036                // contention as a red e2e and blocking the run on it.
7037                self.state.save()?;
7038                return Ok(());
7039            }
7040            last.e2e_deferred = false;
7041        }
7042        let last = &self.state.reviews[round_idx];
7043        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7044
7045        match last.e2e_status() {
7046            E2eStatus::Failed => {
7047                let red: Vec<String> = last
7048                    .e2e
7049                    .iter()
7050                    .filter(|o| !o.ok())
7051                    .map(|o| {
7052                        format!(
7053                            "`{}` -> {:?}\n{}",
7054                            o.command,
7055                            o.code,
7056                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7057                        )
7058                    })
7059                    .collect();
7060                let can_repair = last.blocking == 0 && last.fix.is_none();
7061                self.state.reviews[round_idx].clean = false;
7062                self.state
7063                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7064                self.state.status = RunStatus::Blocked;
7065                if repairable
7066                    && can_repair
7067                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7068                {
7069                    self.state.status = RunStatus::Reviewing;
7070                }
7071            }
7072            // `needs_catchup_run` above already retried once this call; if
7073            // it is still blocked, this is magi's own admission it could
7074            // not get a command to run, never a verdict on the patch — the
7075            // run is left exactly where a later reentry can retry again.
7076            E2eStatus::ResourceBlocked => {
7077                self.state.event(
7078                    "review",
7079                    format!(
7080                        "{why}; e2e could not run (resource unavailable); not \
7081                         deciding yet"
7082                    ),
7083                );
7084            }
7085            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7086                self.state.event(
7087                    "review",
7088                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7089                );
7090                self.record_contested_handoff();
7091                self.state.status = RunStatus::Gating;
7092            }
7093        }
7094        self.state.save()?;
7095        Ok(())
7096    }
7097
7098    // ----------------------------------------------------------------- gate
7099
7100    async fn gate(&mut self) -> Result<()> {
7101        // Judged by the review record itself, not by `status`: a solo
7102        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7103        // reentry (see `judge`), and trusting it here is exactly how a run
7104        // that exhausted its review budget got gated and merged a second
7105        // time around. `review_conclusion` recomputes the review loop's own
7106        // verdict from the round records themselves — `Gating` for a clean
7107        // round or a hand-off (see `stop_reviewing`), anything else means the
7108        // loop is still going or genuinely blocked.
7109        // A base the winner could not be replayed onto is a decision, not a
7110        // round: there is no landing tree to gate. Read as its own record for
7111        // the same reason the review verdict is.
7112        if self.state.status == RunStatus::Failed
7113            || self
7114                .state
7115                .base_sync
7116                .as_ref()
7117                .is_some_and(|s| s.conflict.is_some())
7118            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7119                != Some(RunStatus::Gating)
7120        {
7121            return Ok(());
7122        }
7123        if self.state.gate_ran {
7124            // `review_loop` derives its conclusion from the clean review
7125            // record on every reentry and therefore puts a completed run back
7126            // in `Gating`. A recorded gate is a stronger, terminal fact:
7127            // retain its original command output (or lack of any, for a repo
7128            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7129            // doc) and restore `Blocked` on a real failure rather than
7130            // pretending the command is still running or running it a second
7131            // time. `gate_ran == false` remains the only shape — unattempted,
7132            // or a resource-blocked retry — that may still need to execute a
7133            // command.
7134            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7135                self.state.status = RunStatus::Blocked;
7136                self.state.save()?;
7137            }
7138            return Ok(());
7139        }
7140        let Some(winner) = self.state.winner().cloned() else {
7141            return Ok(());
7142        };
7143        self.state.status = RunStatus::Gating;
7144        let mut outcomes = self.run_gate(&winner).await?;
7145        loop {
7146            // A resource-blocked outcome means the gate command never actually
7147            // ran - the shared build cache could not be acquired or confirmed
7148            // fresh in time - which is evidence about the machine, not about
7149            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7150            // Recording it as a red gate would mark a run `Blocked` on nothing
7151            // but contention magi has already logged; leaving `self.state.gate`
7152            // empty and `self.state.gate_ran` false instead keeps the shape
7153            // this function already treats as "still needs to run" (see the
7154            // early-return above), so the next call retries the command
7155            // rather than concluding anything.
7156            if verify_inconclusive(&outcomes) {
7157                self.state.save()?;
7158                return Ok(());
7159            }
7160            if outcomes.iter().all(CommandOutcome::ok) {
7161                break;
7162            }
7163            match self.gate_fix_round(&winner, &outcomes).await? {
7164                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7165                GateFix::Stop => break,
7166                GateFix::Defer => {
7167                    self.state.save()?;
7168                    return Ok(());
7169                }
7170            }
7171        }
7172        let passed = outcomes.iter().all(CommandOutcome::ok);
7173        self.state.gate = outcomes;
7174        self.state.gate_ran = true;
7175        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7176        if !passed {
7177            self.state.status = RunStatus::Blocked;
7178            let spent = self.state.gate_fixes.len();
7179            self.state.event(
7180                "gate",
7181                if spent == 0 {
7182                    "gate failed; not merging".to_owned()
7183                } else {
7184                    format!("gate failed after {spent} gate-fix round(s); not merging")
7185                },
7186            );
7187        }
7188        self.state.save()?;
7189        Ok(())
7190    }
7191
7192    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7193    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7194    /// after review is clean and never on a candidate awaiting judging.
7195    ///
7196    /// Never fails the run: a non-zero exit or timeout is a warning and a
7197    /// recorded outcome, and the gate remains the single arbiter. Nothing
7198    /// configured means nothing happens - no event, no commit. `commit_all`
7199    /// commits any leftover change under the neutral identity and returns
7200    /// `false` when the tree is clean, so no empty commit is ever made.
7201    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7202        let commands = state.config.verify.pre_gate.clone();
7203        if commands.is_empty() {
7204            return;
7205        }
7206        let shell = state.config.shell();
7207        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7208        let (outcomes, _) = run_commands(
7209            state,
7210            "pre_gate",
7211            "pre_gate",
7212            0,
7213            &shell,
7214            &commands,
7215            &winner.worktree,
7216            timeout,
7217        )
7218        .await;
7219        for o in &outcomes {
7220            if !o.ok() {
7221                tracing::warn!(
7222                    "pre_gate `{}` failed ({:?}); the gate decides",
7223                    o.command,
7224                    o.code
7225                );
7226            }
7227            state.event(
7228                "pre_gate",
7229                format!(
7230                    "`{}` -> {}",
7231                    o.command,
7232                    if o.ok() {
7233                        "pass".to_owned()
7234                    } else {
7235                        format!(
7236                            "FAIL ({:?})\n{}",
7237                            o.code,
7238                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7239                        )
7240                    }
7241                ),
7242            );
7243        }
7244        state.pre_gate = outcomes;
7245        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7246            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7247                Ok(head) => {
7248                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7249                    state.pre_gate_commit = Some(head);
7250                }
7251                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7252            },
7253            Ok(false) => {}
7254            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7255        }
7256        if let Err(e) = state.save() {
7257            tracing::warn!("could not persist the pre_gate record: {e:#}");
7258        }
7259    }
7260
7261    /// Run `verify.gate` once against the winner's current tree, logging one
7262    /// event per command. Empty when nothing is configured.
7263    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7264        let wait = self.repository_wait();
7265        run_landing_gate(&mut self.state, winner, &wait).await
7266    }
7267
7268    /// One bounded fix round for a failing gate.
7269    ///
7270    /// The fixer is told the failure came from the gate itself, not from a
7271    /// reviewer, and is shown the failed commands, their exit codes and a tail
7272    /// of their output - whatever `[verify].gate` holds, nothing here knows
7273    /// what those commands run. Only a normal non-zero exit that printed
7274    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7275    /// command or a full disk says nothing about the code, and a fixer sent
7276    /// after it can only appease the machine. The round is judged by what git
7277    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7278    /// again before the gate does, so a fix cannot trade a green gate for a
7279    /// red e2e unnoticed.
7280    async fn gate_fix_round(
7281        &mut self,
7282        winner: &Candidate,
7283        outcomes: &[CommandOutcome],
7284    ) -> Result<GateFix> {
7285        let cap = self.state.config.graph.gate_fix_rounds;
7286        let spent = self.state.gate_fixes.len();
7287        if spent >= cap {
7288            if cap > 0 {
7289                self.state.event(
7290                    "gate",
7291                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7292                );
7293            }
7294            return Ok(GateFix::Stop);
7295        }
7296        if !gate_fixable(outcomes) {
7297            self.state.event(
7298                "gate",
7299                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7300                 command or similar); not spending a fix round on it",
7301            );
7302            return Ok(GateFix::Stop);
7303        }
7304        let min_free = self.state.config.disk.min_free_bytes;
7305        if min_free > 0 {
7306            match crate::disk::free_bytes(&winner.worktree) {
7307                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7308                Ok(free) => {
7309                    self.state.event(
7310                        "gate",
7311                        format!(
7312                            "only {free} bytes free ({min_free} required by `[disk] \
7313                             min_free_bytes`); not spending a fix round on a failure the disk \
7314                             may explain"
7315                        ),
7316                    );
7317                    return Ok(GateFix::Stop);
7318                }
7319                Err(e) => {
7320                    self.state.event(
7321                        "gate",
7322                        format!("free disk space could not be measured ({e:#}); no fix round"),
7323                    );
7324                    return Ok(GateFix::Stop);
7325                }
7326            }
7327        }
7328
7329        let attempt = spent + 1;
7330        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7331        let base = self.landing_base();
7332        let fix_prompt = prompt::gate_fix_in(
7333            &self.state.instruction,
7334            &failed,
7335            attempt,
7336            cap,
7337            &self.state.config.graph.language,
7338            &self.state.config.graph.github_language,
7339        );
7340        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7341        let sessions = self.state.config.graph.sessions;
7342        let artifacts = self.merge_repair_artifacts();
7343        self.state.event(
7344            "gate",
7345            format!("gate failed; gate-fix round {attempt} of {cap}"),
7346        );
7347        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7348        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7349        // Persist before invoking the fixer: it may commit and then be
7350        // interrupted before its result (or the following e2e) is saved.
7351        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7352            head: before.clone(),
7353            passed: false,
7354            outcomes: Vec::new(),
7355        });
7356        self.state.save()?;
7357        let (_, seat, out) = self
7358            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7359                prompt: fix_prompt.clone(),
7360                spec,
7361                seat,
7362                cwd: winner.worktree.clone(),
7363                timeout,
7364                allow_write: true,
7365                sessions,
7366                artifacts: artifacts.clone(),
7367                stem: format!("gate-fix-{attempt}"),
7368                handover: None,
7369            })
7370            .await;
7371        // The fixer was never asked: no round is spent (nothing is recorded)
7372        // and a later reentry asks the same agent again.
7373        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7374            return Ok(GateFix::Defer);
7375        }
7376        let mut record = GateFixRecord {
7377            agent: seat.agent.clone(),
7378            failed,
7379            notes: String::new(),
7380            committed: false,
7381            error: None,
7382        };
7383        match out {
7384            AgentOutcome::Ok(o) => {
7385                // A missing report is not a failed fix: the round is judged
7386                // by the tree below, and the report only carries prose.
7387                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7388                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7389                }
7390            }
7391            AgentOutcome::Dropped(_) => {
7392                record.error = Some("the CLI dropped the stream".to_owned());
7393            }
7394            AgentOutcome::Quota(o) => {
7395                self.state.quota.push(QuotaLoss {
7396                    seat: seat.key.clone(),
7397                    node: "gate-fix".to_owned(),
7398                    at: Timestamp::now(),
7399                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7400                });
7401                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7402            }
7403            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7404        }
7405        self.state.seats.insert(seat.key.clone(), seat);
7406        let coordination = self.coordinate(&winner.worktree).await?;
7407        if let Ok(r) = git::rescue_commit(
7408            &winner.worktree,
7409            &format!("magi: gate fix {attempt} (uncommitted work)"),
7410        )
7411        .await
7412        {
7413            self.state.note_withheld("gate-fix", &r.withheld);
7414        }
7415        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7416        record.committed = after != before;
7417        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7418        let note = record.error.clone();
7419        self.state.gate_fixes.push(record);
7420        self.state.save()?;
7421        if !changed {
7422            self.state.event(
7423                "gate",
7424                match note {
7425                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7426                    None => format!("gate-fix round {attempt}: the tree did not change"),
7427                },
7428            );
7429            return Ok(GateFix::Stop);
7430        }
7431        self.state.event(
7432            "gate",
7433            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7434        );
7435
7436        drop(coordination);
7437        let wait = self.repository_wait();
7438        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7439    }
7440
7441    // ---------------------------------------------------------------- merge
7442
7443    /// One read-only rewrite by the summary's author; a text the gate still
7444    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7445    /// applies on their word or on silence. `None` means the run is parked on
7446    /// that question and nothing may be pushed or opened yet.
7447    async fn guarded_pr_message(
7448        &mut self,
7449        winner: &Candidate,
7450        facts: Option<&BranchFacts>,
7451        posting: bool,
7452    ) -> Result<Option<PrMessage>> {
7453        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7454        let head = match final_head.as_deref() {
7455            Some(h) => crate::pr_outcome::FinalHead::At(h),
7456            None => crate::pr_outcome::FinalHead::Unknown,
7457        };
7458        // Only a pull request that is about to be posted pays for a summary.
7459        let summarized = if posting {
7460            Some(self.summarize_change(winner, final_head.as_deref()).await)
7461        } else {
7462            None
7463        };
7464        // A busy shared resource is not a permanent cause: publishing an
7465        // "unavailable" description would stick (a later head move never
7466        // edits the body). Stop before anything is pushed or opened; the
7467        // run is `Blocked` and a resume asks the summary again.
7468        if self.state.resource_block.is_some() {
7469            self.state.event(
7470                "pr-summary",
7471                "a shared resource was busy; nothing was published, resume to summarize again",
7472            );
7473            return Ok(None);
7474        }
7475        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7476            anyhow::bail!(
7477                "the final commit moved during public description generation; retry before publishing"
7478            );
7479        }
7480        let note = match (&summarized, &self.state.change_summary) {
7481            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7482            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7483            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7484            (None, _) => ChangeNote::NotAsked,
7485        };
7486        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7487        if !posting {
7488            let identity = crate::scrub::Identity::current();
7489            return Ok(Some(PrMessage {
7490                title: crate::scrub::scrub(&pr.title, &identity),
7491                body: crate::scrub::scrub(&pr.body, &identity),
7492            }));
7493        }
7494        // This very text was already rejected and asked about: no second
7495        // judge call, rewrite or question, just carry the record on.
7496        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7497        if let Some(known) = self
7498            .state
7499            .github_text
7500            .clone()
7501            .filter(|g| g.fingerprint == raw_fp)
7502        {
7503            return self.settle_withheld(winner, &pr, known).await;
7504        }
7505        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7506        let mut violations =
7507            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7508        let identity = crate::scrub::Identity::current();
7509        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7510            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7511            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7512        {
7513            violations.push(crate::github_text::Violation::SensitiveData);
7514        }
7515        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7516        // only a language violation starts one; sensitive data alone goes
7517        // straight to the owner, with the redacted text as the fallback.
7518        let has_language = violations
7519            .iter()
7520            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7521        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7522            self.state.event(
7523                "github-text",
7524                if has_language {
7525                    format!("description rejected: {violations:?}; requesting one rewrite")
7526                } else {
7527                    format!("description rejected: {violations:?}; asking the owner")
7528                },
7529            );
7530            let mut rewritten = false;
7531            let mut rejected_rewrite = None;
7532            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7533            {
7534                let output = text;
7535                if let Some(title) = summary_title(&output) {
7536                    // Judge the redacted candidate that will actually be posted.
7537                    // Sensitive data alone must not discard a useful rewrite.
7538                    let raw_body = summary_without_title(&output);
7539                    let clean_title = crate::scrub::scrub(&title, &identity);
7540                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7541                    let redacted = clean_title != title || body != raw_body;
7542                    let title = clean_title;
7543                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7544                    if !complete {
7545                        self.state.event(
7546                            "github-text",
7547                            "description rewrite omitted required findings",
7548                        );
7549                    }
7550                    if !body.trim().is_empty() {
7551                        body.push_str("\n\n");
7552                        body.push_str(&public_pr_evidence(&self.state, head));
7553                    }
7554                    let marker = format!("magi:run/{}", self.state.id);
7555                    if !body.trim().is_empty() && !body.contains(&marker) {
7556                        body.push_str(&format!("\n\n{marker}"));
7557                    }
7558                    let rewrite = PrMessage {
7559                        title: title.clone(),
7560                        body: body.clone(),
7561                    };
7562                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7563                    let mut failed = crate::github_text::check_for(
7564                        &self.state.config.graph,
7565                        &title,
7566                        &body,
7567                        redecision,
7568                    );
7569                    if (!crate::github_text::shareable(&title)
7570                        || !crate::github_text::shareable(&body))
7571                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7572                    {
7573                        failed.push(crate::github_text::Violation::SensitiveData);
7574                    }
7575                    if complete
7576                        && !title.trim().is_empty()
7577                        && !body.trim().is_empty()
7578                        && !failed.is_empty()
7579                    {
7580                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7581                            title: title.clone(),
7582                            body: body.clone(),
7583                            failed: failed
7584                                .iter()
7585                                .map(|v| crate::github_text::category(*v).to_owned())
7586                                .collect(),
7587                        });
7588                    }
7589                    if complete
7590                        && !title.trim().is_empty()
7591                        && !raw_body.trim().is_empty()
7592                        && failed.is_empty()
7593                    {
7594                        decision = redecision;
7595                        pr = PrMessage { title, body };
7596                        rewritten = true;
7597                        self.state
7598                            .event("github-text", "description rewrite passed");
7599                        if redacted {
7600                            self.state.event(
7601                                "github-text",
7602                                "sensitive data removed from description rewrite",
7603                            );
7604                        }
7605                    }
7606                }
7607            }
7608            if !rewritten {
7609                self.state.event(
7610                    "github-text",
7611                    "description rewrite unavailable or rejected; asking the owner",
7612                );
7613                if let Some(w) =
7614                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7615                {
7616                    // A different text than the one asked about before: that
7617                    // question no longer describes anything.
7618                    self.retire_github_text_question("the withheld text changed");
7619                    let record = crate::run::GithubTextAsk {
7620                        rewrite: rejected_rewrite,
7621                        fingerprint: raw_fp,
7622                        title: w.title,
7623                        body: w.body,
7624                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7625                        question: None,
7626                        asks: 0,
7627                        resolved: false,
7628                        chosen_title: None,
7629                        choice: None,
7630                    };
7631                    return self.settle_withheld(winner, &pr, record).await;
7632                }
7633            }
7634        }
7635        let (title, body) =
7636            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7637        Ok(Some(PrMessage { title, body }))
7638    }
7639
7640    /// Describe the whole change from its final diff in either run mode.
7641    ///
7642    /// The summary is tied to the final commit and to the merge base of the
7643    /// resolved remote base; a record for any other pair is dropped and the
7644    /// agent asked again, so a description never outlives the diff it was
7645    /// written from. The seat is read-only and the prompt tells it to read the
7646    /// diff and the code around it. `[roles] synthesizer` selects the agents
7647    /// (unset: the default pick), each asked at most once, advancing on an
7648    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7649    /// carries the reason it is unavailable; nothing is invented then.
7650    async fn summarize_change(
7651        &mut self,
7652        winner: &Candidate,
7653        head: Option<&str>,
7654    ) -> std::result::Result<(), String> {
7655        let Some(head) = head else {
7656            return Err("the final commit could not be read".to_owned());
7657        };
7658        let remote = self.state.config.merge.remote.clone();
7659        let tracking = format!("{remote}/{}", self.state.base_branch);
7660        if !git::rev_exists(&winner.worktree, &tracking).await {
7661            return Err(format!("the remote base `{tracking}` could not be read"));
7662        }
7663        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7664            Ok(b) if !b.is_empty() => b,
7665            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7666        };
7667        let evidence = crate::pr_outcome::summary_evidence(
7668            &self.state,
7669            crate::pr_outcome::FinalHead::At(head),
7670        );
7671        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7672        if self
7673            .state
7674            .change_summary
7675            .as_ref()
7676            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7677        {
7678            return Ok(());
7679        }
7680        if self.state.change_summary.take().is_some() {
7681            self.state.event(
7682                "pr-summary",
7683                "the final diff or evidence changed; the earlier change summary was discarded",
7684            );
7685        }
7686        // Save the source snapshot even if no generation agent is available.
7687        if let Err(e) = write_artifact(
7688            &self.state,
7689            &format!("public-description-{fingerprint}.md"),
7690            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7691        ) {
7692            tracing::warn!("could not save public description evidence: {e:#}");
7693        }
7694        let chain = match agent::pick_chain(
7695            &self.state.config.agents,
7696            self.state.config.roles.synthesizer.as_ref(),
7697            &agent::installed,
7698            "synthesizer",
7699        ) {
7700            Ok(chain) => chain,
7701            Err(e) => {
7702                tracing::warn!("no agent for the change summary: {e:#}");
7703                self.state.event("pr-summary", "no summary agent available");
7704                return Err("no agent was available to summarize it".to_owned());
7705            }
7706        };
7707        let subjects = git::subjects(&winner.worktree, &base, head)
7708            .await
7709            .unwrap_or_default();
7710        let stat = git::diff_stat(&winner.worktree, &base, head)
7711            .await
7712            .unwrap_or_default();
7713        let stat: String = stat.chars().take(6_000).collect();
7714        let mut prompt = crate::prompt::change_summary_in(
7715            &base,
7716            head,
7717            &subjects,
7718            stat.trim(),
7719            &self.state.config.graph.github_language,
7720        );
7721        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7722        let sessions = self.state.config.graph.sessions;
7723        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7724        let prompts = self.state.config.prompts.clone();
7725        let cache = self.state.config.cache_dir();
7726        let run = self.state.id.clone();
7727        for (i, spec) in chain.into_iter().enumerate() {
7728            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7729            let github_spec = match agent::for_github_text(
7730                &spec,
7731                &self.state.config.graph.github_language,
7732                &winner.worktree,
7733            ) {
7734                Ok(spec) => spec,
7735                Err(e) => {
7736                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7737                    continue;
7738                }
7739            };
7740            let job = SeatJob {
7741                spec: github_spec,
7742                seat,
7743                cwd: winner.worktree.clone(),
7744                prompt: prompt.clone(),
7745                timeout,
7746                allow_write: false,
7747                sessions,
7748                artifacts: agent::artifacts_dir(&self.state.dir()),
7749                stem: if i == 0 {
7750                    "change-summary".to_owned()
7751                } else {
7752                    format!("change-summary-{}", spec.id)
7753                },
7754                handover: None,
7755            };
7756            let ctx = WaveCtx {
7757                run: &run,
7758                node: "pr-summary",
7759                prompts: &prompts,
7760                cache: cache.as_deref(),
7761                round: None,
7762                carry_seats: false,
7763                pauses: Some((&self.pause, &self.interrupt)),
7764            };
7765            let (seat, outcome) =
7766                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7767            self.state.seats.insert(seat.key.clone(), seat);
7768            let text = match outcome {
7769                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7770                AgentOutcome::ResourceBlocked(why) => {
7771                    // No agent was asked: the run stops resumably before
7772                    // anything is published (see `guarded_pr_message`).
7773                    if self.state.resource_block.is_none() {
7774                        self.state.resource_block = Some(crate::run::ResourceBlock {
7775                            cause: crate::run::ResourceBlockCause::Other,
7776                            node: "pr-summary".to_owned(),
7777                            kind: "shared resource".to_owned(),
7778                            detail: why,
7779                            at: Timestamp::now(),
7780                        });
7781                    }
7782                    return Err("a shared resource was busy".to_owned());
7783                }
7784                _ => continue,
7785            };
7786            let blind = &self.state.config.blind;
7787            let Some(title) = summary_title(&text) else {
7788                continue;
7789            };
7790            let body: String = summary_without_title(&text)
7791                .lines()
7792                .filter(|l| !l.contains("magi:run/"))
7793                .collect::<Vec<_>>()
7794                .join("\n");
7795            let body = blind::sanitize_prose(body.trim(), blind);
7796            let title = blind::sanitize_prose(&title, blind);
7797            if body.trim().is_empty() || title.trim().is_empty() {
7798                continue;
7799            }
7800            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7801                continue;
7802            }
7803            // The branch must still be the one the agent was shown.
7804            if git::rev_parse(&winner.worktree, "HEAD")
7805                .await
7806                .ok()
7807                .as_deref()
7808                != Some(head)
7809            {
7810                return Err("the branch moved while the summary was written".to_owned());
7811            }
7812            if git::merge_base(&winner.worktree, &tracking, head)
7813                .await
7814                .ok()
7815                .as_deref()
7816                != Some(base.as_str())
7817                || crate::pr_outcome::summary_evidence(
7818                    &self.state,
7819                    crate::pr_outcome::FinalHead::At(head),
7820                ) != evidence
7821            {
7822                return Err("the summary evidence changed while it was written".to_owned());
7823            }
7824            self.state.change_summary = Some(crate::run::ChangeSummary {
7825                head: head.to_owned(),
7826                base,
7827                evidence: fingerprint,
7828                title,
7829                body,
7830            });
7831            self.state.event(
7832                "pr-summary",
7833                format!("change summary written by {} for the final diff", spec.id),
7834            );
7835            if let Err(e) = self.state.save() {
7836                tracing::warn!("could not save the change summary: {e:#}");
7837            }
7838            return Ok(());
7839        }
7840        self.state
7841            .event("pr-summary", "no agent produced a change summary");
7842        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7843    }
7844
7845    /// One read-only rewrite request for a withheld description. The
7846    /// implementer's own conversation is tried first when it exists and can be
7847    /// resumed; then the `[roles] language_judge` chain (or the default pick)
7848    /// writes it in a separate conversation, skipping the implementer's agent.
7849    /// The prompt carries the whole text, so no agent needs any memory. Each
7850    /// agent is asked at most once and the chain advances on an error, quota,
7851    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
7852    /// that has run out of quota must not end the rewrite. Nothing is recorded
7853    /// as a handover.
7854    async fn rewrite_pr_text(
7855        &mut self,
7856        winner: &Candidate,
7857        pr: &PrMessage,
7858        violations: &[crate::github_text::Violation],
7859    ) -> Option<String> {
7860        let prompt = format!(
7861            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7862            pr.title, pr.body
7863        );
7864        let sessions = self.state.config.graph.sessions;
7865        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7866        // (spec, the resumed implementer's seat). A fallback seat is created
7867        // only when its turn comes, so an unused fallback leaves no record.
7868        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7869        let key = format!("impl-{}", winner.label);
7870        if let Some(spec) = self
7871            .state
7872            .config
7873            .agents
7874            .iter()
7875            .find(|a| a.id == winner.agent)
7876            .cloned()
7877            && let Some(existing) = self.state.seats.get(&key)
7878            && existing.agent == spec.id
7879            && agent::has_session(spec.kind, existing, sessions)
7880        {
7881            attempts.push((spec, Some(existing.clone())));
7882        }
7883        match agent::pick_chain(
7884            &self.state.config.agents,
7885            self.state.config.roles.language_judge.as_ref(),
7886            &agent::installed,
7887            "language_judge",
7888        ) {
7889            Ok(chain) => {
7890                for spec in chain {
7891                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7892                        continue;
7893                    }
7894                    attempts.push((spec, None));
7895                }
7896            }
7897            Err(e) if attempts.is_empty() => {
7898                tracing::warn!("no agent for the description rewrite: {e:#}");
7899                self.state
7900                    .event("github-text", "no rewrite agent available");
7901                return None;
7902            }
7903            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7904        }
7905        let prompts = self.state.config.prompts.clone();
7906        let cache = self.state.config.cache_dir();
7907        let run = self.state.id.clone();
7908        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7909            let resumed = resumed_seat.is_some();
7910            let seat = resumed_seat.unwrap_or_else(|| {
7911                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7912            });
7913            let github_spec = match agent::for_github_text(
7914                &spec,
7915                &self.state.config.graph.github_language,
7916                &winner.worktree,
7917            ) {
7918                Ok(spec) => spec,
7919                Err(e) => {
7920                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7921                    continue;
7922                }
7923            };
7924            let job = SeatJob {
7925                spec: github_spec,
7926                seat,
7927                cwd: winner.worktree.clone(),
7928                prompt: prompt.clone(),
7929                timeout: retry_budget(timeout_full, resumed),
7930                allow_write: false,
7931                sessions,
7932                artifacts: agent::artifacts_dir(&self.state.dir()),
7933                stem: if i == 0 {
7934                    "github-text-rewrite".to_owned()
7935                } else {
7936                    format!("github-text-rewrite-{}", spec.id)
7937                },
7938                handover: None,
7939            };
7940            let ctx = WaveCtx {
7941                run: &run,
7942                node: "github-text",
7943                prompts: &prompts,
7944                cache: cache.as_deref(),
7945                round: None,
7946                carry_seats: false,
7947                pauses: Some((&self.pause, &self.interrupt)),
7948            };
7949            let (seat, outcome) =
7950                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7951            self.state.seats.insert(seat.key.clone(), seat);
7952            match outcome {
7953                AgentOutcome::Ok(o)
7954                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7955                {
7956                    return Some(o.text);
7957                }
7958                AgentOutcome::ResourceBlocked(_) => return None,
7959                _ => {}
7960            }
7961        }
7962        None
7963    }
7964
7965    /// Abandon the standing posting-gate question, if any.
7966    fn retire_github_text_question(&mut self, why: &str) {
7967        let Some(id) = self
7968            .state
7969            .github_text
7970            .as_ref()
7971            .and_then(|g| g.question.clone())
7972        else {
7973            return;
7974        };
7975        let _ = ask::Questions::open().update(&id, |q| {
7976            q.abandon(why);
7977            Ok(())
7978        });
7979    }
7980
7981    /// Carry a withheld text to its end: file the question, park while it is
7982    /// open, and on an answer, silence or timeout pick the text to post. The
7983    /// decision is saved before it is used, so it applies once.
7984    async fn settle_withheld(
7985        &mut self,
7986        winner: &Candidate,
7987        pr: &PrMessage,
7988        mut g: crate::run::GithubTextAsk,
7989    ) -> Result<Option<PrMessage>> {
7990        use crate::github_text as gt;
7991        let store = ask::Questions::open();
7992        let timeout = self.state.config.graph.answer_timeout;
7993        if !g.resolved {
7994            let mut standing = g
7995                .question
7996                .as_ref()
7997                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7998            if let Some(q) = standing.as_ref()
7999                && q.status.open()
8000                && gt::expired(q, timeout)
8001            {
8002                let why = format!("no answer within {}s of asking", timeout.max(1));
8003                standing = store
8004                    .update(&q.id, |q| {
8005                        q.abandon(&why);
8006                        Ok(())
8007                    })
8008                    .ok()
8009                    .map(|(q, ())| q);
8010            }
8011            let mut ask_again: Option<Option<Vec<String>>> = None;
8012            match standing {
8013                // Recorded but never written (a stop between the two saves),
8014                // or removed: ask again without spending the retry.
8015                None if g.asks <= 1 => {
8016                    g.asks = 0;
8017                    ask_again = Some(None);
8018                }
8019                None => {
8020                    g.resolved = true;
8021                    g.chosen_title = None;
8022                    g.choice = Some(GithubTextChoice::Fallback);
8023                }
8024                Some(q) if q.status.open() => {
8025                    self.park_on_github_text(&g)?;
8026                    return Ok(None);
8027                }
8028                Some(q) => match gt::read_reply(&q) {
8029                    gt::Reply::Fallback => {
8030                        g.resolved = true;
8031                        g.chosen_title = None;
8032                        g.choice = Some(GithubTextChoice::Fallback);
8033                    }
8034                    gt::Reply::Original => {
8035                        g.resolved = true;
8036                        g.chosen_title = None;
8037                        g.choice = Some(GithubTextChoice::Original);
8038                    }
8039                    gt::Reply::Rewrite => {
8040                        g.resolved = true;
8041                        g.chosen_title = None;
8042                        g.choice = Some(if g.rewrite.is_some() {
8043                            GithubTextChoice::Rewrite
8044                        } else {
8045                            GithubTextChoice::Fallback
8046                        });
8047                    }
8048                    gt::Reply::Title(text) => {
8049                        let first = text
8050                            .lines()
8051                            .map(str::trim)
8052                            .find(|l| !l.is_empty())
8053                            .unwrap_or("");
8054                        let decision = if first.is_empty() {
8055                            None
8056                        } else {
8057                            let candidate = PrMessage {
8058                                title: first.to_owned(),
8059                                body: String::new(),
8060                            };
8061                            self.judge_pr_language(&winner.worktree, &candidate).await
8062                        };
8063                        match gt::vet_title(
8064                            &text,
8065                            decision,
8066                            self.state.config.graph.github_english(),
8067                        ) {
8068                            Ok(title) => {
8069                                g.resolved = true;
8070                                g.chosen_title = Some(title);
8071                                g.choice = Some(GithubTextChoice::MyTitle);
8072                            }
8073                            Err(cats) if g.asks < 2 => {
8074                                ask_again = Some(Some(
8075                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8076                                ))
8077                            }
8078                            Err(_) => {
8079                                g.resolved = true;
8080                                g.chosen_title = None;
8081                                g.choice = Some(GithubTextChoice::Fallback);
8082                            }
8083                        }
8084                    }
8085                },
8086            }
8087            if let Some(rejected) = ask_again {
8088                // A retry (after a rejected replacement title) is still about
8089                // the original text: the same ORIGINAL / FALLBACK comparison.
8090                let w = gt::Withheld {
8091                    title: g.title,
8092                    body: g.body,
8093                    categories: g.categories.clone(),
8094                };
8095                // Named by the original text's fingerprint alone, so a resume
8096                // rewrites the same file and the detail stays identical.
8097                let artifact = match crate::run::write_artifact(
8098                    &self.state,
8099                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8100                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8101                ) {
8102                    Ok(p) => Some(p),
8103                    Err(e) => {
8104                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8105                        None
8106                    }
8107                };
8108                let (detail, comparison) = gt::question_content(
8109                    &w,
8110                    &pr.title,
8111                    &pr.body,
8112                    &self.state.id,
8113                    rejected.as_deref(),
8114                    artifact.as_deref(),
8115                    g.rewrite.as_ref(),
8116                );
8117                let mut q = ask::Question::new(
8118                    self.state.id.clone(),
8119                    gt::ASK_NODE.to_owned(),
8120                    gt::ASK_SEAT.to_owned(),
8121                    gt::question_summary(&self.state.config.graph.language, &w),
8122                    detail,
8123                    gt::question_choices(&w, g.rewrite.as_ref()),
8124                );
8125                q.posting_comparison = Some(comparison);
8126                q.answer_timeout = timeout;
8127                // Recorded before it is written: a stop in between leaves a
8128                // record naming a question the resume then files again.
8129                g.question = Some(q.id.clone());
8130                g.asks += 1;
8131                self.state.github_text = Some(g.clone());
8132                self.state.event(
8133                    "github-text",
8134                    format!("asking the owner about the withheld text ({})", q.short()),
8135                );
8136                self.state.save()?;
8137                store
8138                    .put_executable(&mut q)
8139                    .context("file the posting-gate question")?;
8140                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8141                    tracing::warn!(
8142                        "could not notify about posting-gate question {}: {e:#}",
8143                        q.short()
8144                    );
8145                }
8146                self.park_on_github_text(&g)?;
8147                return Ok(None);
8148            }
8149        }
8150        let resolved = g.chosen_title.clone();
8151        let original = g.choice == Some(GithubTextChoice::Original);
8152        // Field names only: the event log never records the text.
8153        self.state.event(
8154            "github-text",
8155            if original {
8156                format!(
8157                    "posting the original text for: {}",
8158                    match (g.title, g.body) {
8159                        (true, true) => "title, description",
8160                        (true, false) => "title",
8161                        _ => "description",
8162                    }
8163                )
8164            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8165                "posting the saved rewrite title and description".to_owned()
8166            } else if resolved.is_some() {
8167                "posting the owner's replacement title".to_owned()
8168            } else {
8169                "posting the neutral text for the withheld fields".to_owned()
8170            },
8171        );
8172        self.state.github_text = Some(g.clone());
8173        self.state.save()?;
8174        if g.choice == Some(GithubTextChoice::Rewrite)
8175            && let Some(r) = g.rewrite.as_ref()
8176        {
8177            return Ok(Some(PrMessage {
8178                title: r.title.clone(),
8179                body: r.body.clone(),
8180            }));
8181        }
8182        let w = gt::Withheld {
8183            title: g.title,
8184            body: g.body,
8185            categories: g.categories.clone(),
8186        };
8187        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8188        if let Some(t) = resolved {
8189            title = t;
8190        }
8191        // Every withheld field is now neutral or vetted English.
8192        let english = gt::LanguageDecision {
8193            title_english: true,
8194            body_english: true,
8195            body_complete: true,
8196        };
8197        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8198        if original {
8199            // The owner's explicit choice overrides the gate for the fields it
8200            // was made for; the others keep what the gate produced.
8201            if g.title {
8202                title = pr.title.clone();
8203            }
8204            if g.body {
8205                body = pr.body.clone();
8206            }
8207        }
8208        Ok(Some(PrMessage { title, body }))
8209    }
8210
8211    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8212        self.state.github_text = Some(g.clone());
8213        self.state.end_pass_parked(ParkCause::Question);
8214        self.state.parked = true;
8215        self.state.event(
8216            "github-text",
8217            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8218        );
8219        self.state.save()
8220    }
8221
8222    /// Ask `[roles] language_judge` about this text and record which source
8223    /// decided; the text itself is never recorded.
8224    async fn judge_pr_language(
8225        &mut self,
8226        cwd: &Path,
8227        pr: &PrMessage,
8228    ) -> Option<crate::github_text::LanguageDecision> {
8229        self.state.config.roles.language_judge.as_ref()?;
8230        if !self.state.config.graph.language_gate() {
8231            return None;
8232        }
8233        let decision =
8234            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8235        self.state.event(
8236            "github-text",
8237            if decision.is_some() {
8238                "language decided by the language judge"
8239            } else {
8240                "language judge unavailable; using built-in heuristics"
8241            },
8242        );
8243        decision
8244    }
8245
8246    async fn merge(&mut self) -> Result<()> {
8247        // Same reasoning as `gate`: ask the review and gate records directly
8248        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8249        // skip can rewrite on reentry to something that no longer says
8250        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8251        // so a hand-off (open findings, green verification) reaches merge
8252        // exactly like a genuinely clean round does.
8253        //
8254        // A run resumed mid-`land` never reaches here at all: `execute`
8255        // recognises `RunStatus::Landing` before it even calls `prep`, and
8256        // routes straight to `run_land` instead. That has to happen a level
8257        // up from this function, not with a check in here, because
8258        // `review_loop`'s own status recomputation (see its doc) runs
8259        // *before* `merge` on every reentry and would otherwise overwrite
8260        // the `Landing` marker with `Gating` before this node ever saw it.
8261        if self
8262            .state
8263            .base_sync
8264            .as_ref()
8265            .is_some_and(|s| s.conflict.is_some())
8266            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8267                != Some(RunStatus::Gating)
8268            // `gate_ran == false` is not "passed" - `gate` leaves it false
8269            // both before it has ever run and when its last attempt was
8270            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8271            // permission to merge on nothing but the review record. Only a
8272            // gate that actually ran may proceed; an intentional local
8273            // review can also finish with verification explicitly not configured.
8274            // `RunState::gate_status` computes that distinction.
8275            || !self.state.gate_allows_progress()
8276        {
8277            return Ok(());
8278        }
8279        // This node's own record, not `status`: `status == Ready` is not
8280        // unique to the harmless `MergeMode::None` path this line was
8281        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8282        // run's PR was closed without merging — and on that run `mode` is
8283        // still `Pr`, so a reentry that fell through here would push and
8284        // open a second pull request. `self.state.merge` is set exactly once
8285        // this node (or `land`) has already produced a verdict, under every
8286        // mode, which is what "already done" actually means here.
8287        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8288            return Ok(());
8289        }
8290        let Some(winner) = self.state.winner().cloned() else {
8291            return Ok(());
8292        };
8293        let repo = self.state.repo.clone();
8294        let base = self.state.base_branch.clone();
8295        let mode = self.state.config.merge.mode;
8296        let style = self.state.config.merge.style;
8297        let facts = if is_review_run(&self.state) {
8298            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8299            let start = review_base(
8300                &repo,
8301                &self.state.config.merge.remote,
8302                &base,
8303                &self.state.base_commit,
8304                &winner.branch,
8305            )
8306            .await;
8307            branch_facts(&repo, &start, &winner.branch).await
8308        } else {
8309            None
8310        };
8311        // `None` when the base could not be freshly read: then an adopted
8312        // pull request's title is left alone.
8313        let leaked = if is_review_run(&self.state) {
8314            leaked_subjects(&self.state, &winner.branch).await
8315        } else {
8316            Some(Vec::new())
8317        };
8318        // `None`: parked on the owner's word about a withheld title or body.
8319        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8320        // the resume arrives here again.
8321        let Some(pr) = self
8322            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8323            .await?
8324        else {
8325            return Ok(());
8326        };
8327        let message = pr.commit_message();
8328
8329        let outcome = match mode {
8330            MergeMode::None => MergeOutcome {
8331                mode,
8332                ok: true,
8333                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8334                empty: false,
8335            },
8336            MergeMode::Pr | MergeMode::Local
8337                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8338            {
8339                MergeOutcome {
8340                    mode,
8341                    ok: false,
8342                    detail: empty_candidate_detail(&self.state, &base),
8343                    empty: true,
8344                }
8345            }
8346            MergeMode::Local => {
8347                let on = git::current_branch(&repo).await?;
8348                if on.as_deref() != Some(base.as_str()) {
8349                    MergeOutcome {
8350                        mode,
8351                        ok: false,
8352                        detail: format!(
8353                            "{} has {} checked out, not the base branch {base}",
8354                            repo.display(),
8355                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8356                        ),
8357                        empty: false,
8358                    }
8359                } else if !git::is_clean(&repo).await? {
8360                    MergeOutcome {
8361                        mode,
8362                        ok: false,
8363                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8364                        empty: false,
8365                    }
8366                } else {
8367                    let out = match style {
8368                        MergeStyle::Merge => {
8369                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8370                        }
8371                        MergeStyle::Squash => {
8372                            git::merge_squash(&repo, &winner.branch, &message).await?
8373                        }
8374                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8375                    };
8376                    MergeOutcome {
8377                        mode,
8378                        ok: out.ok(),
8379                        detail: if out.ok() { out.stdout } else { out.stderr },
8380                        empty: false,
8381                    }
8382                }
8383            }
8384            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8385                self.state.pr_publication_pending = true;
8386                self.state.save()?;
8387                let out =
8388                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8389                self.state.pr_publication_pending = out.is_err();
8390                MergeOutcome {
8391                    mode,
8392                    ok: out.is_ok(),
8393                    detail: match out {
8394                        Ok(url) => url,
8395                        Err(e) => format!("{e:#}"),
8396                    },
8397                    empty: false,
8398                }
8399            }
8400            MergeMode::Pr => {
8401                let remote = self.state.config.merge.remote.clone();
8402                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8403                if !pushed.ok() {
8404                    MergeOutcome {
8405                        mode,
8406                        ok: false,
8407                        detail: pushed.stderr,
8408                        empty: false,
8409                    }
8410                } else {
8411                    // A retry or resume of a run whose branch already has an
8412                    // open pull request adopts it rather than failing on a
8413                    // duplicate. Only this winner branch into this base:
8414                    // `branch_for` derives the name from the run id, so a
8415                    // different run's pull request never matches.
8416                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8417                    let out = match pr_merge_plan(found) {
8418                        PrPlan::Create => {
8419                            gh_pr_create(
8420                                &winner.worktree,
8421                                &base,
8422                                &winner.branch,
8423                                &pr.title,
8424                                &pr.body,
8425                            )
8426                            .await
8427                        }
8428                        PrPlan::Adopt { url, title } => {
8429                            self.state
8430                                .event("merge", format!("Pr: adopted open pull request {url}"));
8431                            if self.state.github_text.as_ref().is_some_and(|g| {
8432                                g.resolved
8433                                    && g.choice == Some(GithubTextChoice::Rewrite)
8434                                    && g.rewrite.is_some()
8435                            }) {
8436                                // The owner chose the entire pair, even if the
8437                                // existing title already matches or was edited.
8438                                land::set_pr_text_verbatim(
8439                                    &winner.worktree,
8440                                    &url,
8441                                    &pr.title,
8442                                    &pr.body,
8443                                )
8444                                .await
8445                                .map(|()| url)
8446                            } else {
8447                                if title != pr.title
8448                                    && (!is_review_run(&self.state)
8449                                        || leaked
8450                                            .as_deref()
8451                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8452                                    && let Err(e) =
8453                                        if self.state.github_text.as_ref().is_some_and(|g| {
8454                                            g.resolved
8455                                                && g.title
8456                                                && g.choice == Some(GithubTextChoice::Original)
8457                                        }) {
8458                                            // The owner chose to post this title unmodified.
8459                                            land::set_pr_title_verbatim(
8460                                                &winner.worktree,
8461                                                &url,
8462                                                &pr.title,
8463                                            )
8464                                            .await
8465                                        } else {
8466                                            land::set_pr_title(
8467                                                &mut self.state,
8468                                                &winner.worktree,
8469                                                &url,
8470                                                &pr.title,
8471                                            )
8472                                            .await
8473                                        }
8474                                {
8475                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8476                                    self.state
8477                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8478                                }
8479                                Ok(url)
8480                            }
8481                        }
8482                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8483                    };
8484                    match out {
8485                        Ok(url) => MergeOutcome {
8486                            mode,
8487                            ok: true,
8488                            detail: url,
8489                            empty: false,
8490                        },
8491                        Err(e) => MergeOutcome {
8492                            mode,
8493                            ok: false,
8494                            detail: e.to_string(),
8495                            empty: false,
8496                        },
8497                    }
8498                }
8499            }
8500        };
8501
8502        self.state.status = match (mode, outcome.ok) {
8503            (MergeMode::None, _) => RunStatus::Ready,
8504            (_, true) if outcome.empty => RunStatus::Ready,
8505            (MergeMode::Pr, true) => RunStatus::Ready,
8506            (_, true) => RunStatus::Merged,
8507            (_, false) => RunStatus::Blocked,
8508        };
8509        self.state.event(
8510            "merge",
8511            format!(
8512                "{:?}: {}",
8513                mode,
8514                outcome.detail.lines().next().unwrap_or("")
8515            ),
8516        );
8517        self.state.merge = Some(outcome);
8518        self.state.save()?;
8519
8520        // The PR is open and the run would historically stop here, leaving the
8521        // operator to watch checks, feed review comments back to a fixer, and
8522        // merge. That was done by hand six times in one session before this
8523        // existed. Opt-in, because merging is the one irreversible thing magi
8524        // can do to a repository.
8525        if self.state.config.graph.land
8526            && mode == MergeMode::Pr
8527            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8528        {
8529            self.run_land().await?;
8530        }
8531        // `run_land` may have left `status` at `Landing` - still waiting on
8532        // CI or the owner's approval, not actually settled - so this has to
8533        // read whatever `status` ended up as here, not the `Merged` this
8534        // function set a few lines up.
8535        self.settle_questions();
8536        Ok(())
8537    }
8538
8539    /// Enter `land`.
8540    ///
8541    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8542    /// a resumed run's re-entry. `land::land` itself is what serialises the
8543    /// two git-mutating moments inside the loop — the rebase push and
8544    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8545    /// to hold a lock across the whole call, and doing so would serialise
8546    /// this run's CI wait against a *different* run's land-approval resume
8547    /// in the same repository, which is exactly the "must not wait on
8548    /// another task" property the daemon's slot-freeing exists to give.
8549    async fn run_land(&mut self) -> Result<()> {
8550        crate::pr_head::ensure_owner(&self.state)?;
8551        let url = self
8552            .state
8553            .pr
8554            .as_ref()
8555            .map(|pr| pr.url.clone())
8556            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8557            .unwrap_or_default();
8558        let url = url.lines().next().unwrap_or("").trim().to_owned();
8559        if !url.starts_with("http") {
8560            return Ok(());
8561        }
8562        // A land failure is not a lost run: the work is on a branch and the
8563        // pull request is open, which is exactly where a human takes over.
8564        loop {
8565            match land::land(&mut self.state, &url).await {
8566                Ok(_)
8567                    if self
8568                        .state
8569                        .merge_repairs
8570                        .last()
8571                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8572                {
8573                    if self.repair_merge_review().await? {
8574                        continue;
8575                    }
8576                }
8577                Ok(pr) if self.state.parked => {
8578                    // `land` already saved the parked marker; nothing here
8579                    // overrides `status` back to a terminal value while an
8580                    // approval is still outstanding.
8581                    let _ = pr;
8582                }
8583                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8584                    self.state.status = RunStatus::Superseded;
8585                    self.state.save()?;
8586                }
8587                Ok(pr) => {
8588                    self.state.status = match pr.state {
8589                        land::PrLifecycle::Merged => RunStatus::Merged,
8590                        _ => RunStatus::Blocked,
8591                    };
8592                    // Downstream of a confirmed merge only - see
8593                    // `bump::should_release_bump`'s own doc for why this one
8594                    // check covers all three of `land`'s success paths.
8595                    // Best-effort: the run already landed, so a failure here
8596                    // (the decision call, `gh`, `cargo`) is recorded and never
8597                    // turns a landed run into a failed one.
8598                    if bump::should_release_bump(self.state.status)
8599                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8600                    {
8601                        // The event is the run's own record. Not-eligible cases
8602                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8603                        // `Err` is a bump that was tried and failed:
8604                        // `after_merge` itself raises the operator notice for
8605                        // that, whether or not a release PR exists yet.
8606                        self.state
8607                            .event("bump", format!("release bump skipped: {e:#}"));
8608                    }
8609                    // Independent of the bump, and best-effort in the same way:
8610                    // findings the merge left open become follow-up tasks.
8611                    if self.state.status == RunStatus::Merged {
8612                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8613                    }
8614                    self.state.save()?;
8615                }
8616                Err(e) => {
8617                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8618                        return Err(e);
8619                    }
8620                    self.state.status = RunStatus::Blocked;
8621                    self.state.event("land", format!("gave up: {e}"));
8622                    self.state.save()?;
8623                }
8624            }
8625            break;
8626        }
8627        Ok(())
8628    }
8629
8630    /// One repair cycle stays under the existing driver/task claim. An
8631    /// interrupted fixer is rolled back and held rather than run twice.
8632    async fn repair_merge_review(&mut self) -> Result<bool> {
8633        use crate::run::MergeRepairPhase as Phase;
8634        crate::pr_head::ensure_owner(&self.state)?;
8635        let i = self.state.merge_repairs.len() - 1;
8636        let request = self.state.merge_repairs[i].clone();
8637        if request.phase == Phase::Fixing {
8638            let restored = match self.state.winner().cloned() {
8639                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8640                None => Err(anyhow::anyhow!("repair has no winning branch")),
8641            };
8642            let why = match restored {
8643                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8644                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8645                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8646            };
8647            self.hold_merge_repair(i, why)?;
8648            return Ok(false);
8649        }
8650        let failure = if i >= self.state.config.graph.land_rounds
8651            || self.state.config.graph.review_rounds == 0
8652        {
8653            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8654        } else {
8655            None
8656        };
8657        if let Some(why) = failure {
8658            self.hold_merge_repair(i, why)?;
8659            return Ok(false);
8660        }
8661        if request.phase == Phase::Requested {
8662            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8663                Ok(pr) => pr,
8664                Err(e) => {
8665                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8666                    return Ok(false);
8667                }
8668            };
8669            self.state.merge_repairs[i].phase = Phase::Fixing;
8670            self.state.land_approval = None;
8671            self.state.land_armed_head = None;
8672            self.state.save()?;
8673            // Serialize the exact records, including finding ids/details,
8674            // reconsideration reasons, final votes and verification output.
8675            let evidence = serde_json::to_string_pretty(&request)?;
8676            let reason = format!(
8677                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8678            );
8679            let budget = self.state.config.graph.land_rounds;
8680            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8681            match result {
8682                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8683                Ok(land::Fixed::Failed(why)) => {
8684                    self.hold_merge_repair(
8685                        i,
8686                        format!(
8687                            "repair failed: {why}; inspect the branch and run an explicit review"
8688                        ),
8689                    )?;
8690                    return Ok(false);
8691                }
8692                Err(e) => {
8693                    self.hold_merge_repair(
8694                        i,
8695                        format!(
8696                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8697                        ),
8698                    )?;
8699                    return Ok(false);
8700                }
8701            }
8702            // Even no change gets a real new reviewer wave. The previous
8703            // evidence remains in the request; a former clean/concluded round
8704            // cannot short-circuit the new review or gate.
8705            self.begin_merge_repair_review(i)?;
8706        }
8707        self.review_loop().await?;
8708        self.gate().await?;
8709        if self.resource_halt()? || self.state.parked {
8710            return Ok(false);
8711        }
8712        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8713            != Some(RunStatus::Gating)
8714            || !self.state.gate_allows_progress()
8715        {
8716            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8717            return Ok(false);
8718        }
8719        // Review and gate may have added commits after the initial repair.
8720        // Publish exactly their final head to the existing PR, with a lease.
8721        let winner = self
8722            .state
8723            .winner()
8724            .cloned()
8725            .context("repair has no winning branch")?;
8726        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8727        if self.state.gate_head.as_deref() != Some(after.as_str())
8728            || !git::is_clean(&winner.worktree).await?
8729        {
8730            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8731            return Ok(false);
8732        }
8733        let expected = self
8734            .state
8735            .pr_pushed_head
8736            .as_deref()
8737            .unwrap_or(&request.head)
8738            .to_owned();
8739        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8740        // Push the checked commit itself: a local ref moving after the check
8741        // must not change which commit the forge receives.
8742        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8743        let refspec = format!("{after}:refs/heads/{public}");
8744        let push = git::git_raw(
8745            &winner.worktree,
8746            &["push", &lease, &self.state.config.merge.remote, &refspec],
8747        )
8748        .await;
8749        match push {
8750            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8751            Ok(out) => {
8752                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8753                return Ok(false);
8754            }
8755            Err(e) => {
8756                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8757                return Ok(false);
8758            }
8759        }
8760        self.state.merge_repairs[i].phase = Phase::Complete;
8761        self.state.status = RunStatus::Landing;
8762        self.state.event(
8763            "repair",
8764            "new review and gate completed; fresh owner merge decision required",
8765        );
8766        self.state.save()?;
8767        Ok(true)
8768    }
8769
8770    fn merge_repair_artifacts(&self) -> PathBuf {
8771        let artifacts = agent::artifacts_dir(&self.state.dir());
8772        match self
8773            .state
8774            .merge_repairs
8775            .last()
8776            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8777        {
8778            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8779            None => artifacts,
8780        }
8781    }
8782
8783    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8784        self.state.reviews.clear();
8785        self.state.gate.clear();
8786        self.state.gate_ran = false;
8787        self.state.gate_head = None;
8788        self.state.gate_fixes.clear();
8789        self.state.gate_repair_e2e = None;
8790        self.state.pre_gate.clear();
8791        self.state.pre_gate_commit = None;
8792        self.state.contested_handoff = None;
8793        self.state.land_reruns = None;
8794        self.state.land_approval = None;
8795        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8796        self.state.status = RunStatus::Reviewing;
8797        self.state.save()
8798    }
8799
8800    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8801        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8802        self.state.merge_repairs[index].failure = Some(why.clone());
8803        self.state.status = RunStatus::Blocked;
8804        self.state.event("repair", why);
8805        self.state.save()
8806    }
8807
8808    // -------------------------------------------------------------- helpers
8809
8810    /// Fetch or create a seat, keeping its conversation across nodes.
8811    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8812        if let Some(existing) = self.state.seats.get(key)
8813            && existing.agent == agent
8814        {
8815            return existing.clone();
8816        }
8817        // A seat that changes agent mints its session id from the agent too,
8818        // like a handover: the old agent's uuid is already taken by the CLI.
8819        let fresh = if self.state.seats.contains_key(key) {
8820            handover_seat(key, agent, self.state.next_seat_seed())
8821        } else {
8822            SeatState::new(key, agent, self.state.seed)
8823        };
8824        self.state.seats.insert(key.to_owned(), fresh.clone());
8825        fresh
8826    }
8827
8828    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8829    /// seat to another roster agent, in which case that agent. Nodes that
8830    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8831    /// reconsideration) must keep talking to whoever answered it, not slip
8832    /// back to the agent that failed it.
8833    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8834        match self.state.seats.get(key) {
8835            Some(s) if s.agent != spec.id => {
8836                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8837            }
8838            _ => spec,
8839        }
8840    }
8841
8842    /// A candidate rendered for judging, with the leak policy applied.
8843    fn view(&self, c: &Candidate) -> CandidateView {
8844        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8845            .unwrap_or_default();
8846        let (patch, _) = blind::sanitize_patch(
8847            &format!("candidate {} patch", c.label),
8848            &raw,
8849            &self.state.config.blind,
8850        );
8851        CandidateView {
8852            label: c.label,
8853            branch: c.branch.clone(),
8854            summary: c.summary.clone(),
8855            stat: c.stat.clone(),
8856            patch,
8857        }
8858    }
8859
8860    /// The full candidate set as prompt text, for seats with no live session.
8861    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8862        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8863        prompt::judge(
8864            "(see above)",
8865            &views,
8866            self.roles.judges.len(),
8867            base_short,
8868            "en",
8869        )
8870    }
8871
8872    /// The final-vote prompt with everything a seat that has no session of its
8873    /// own needs: the candidates, the seat's own ranking and reasons, and the
8874    /// anonymised deliberation it took part in (only when there was one, so a
8875    /// handed-over seat never sees more than the seat it replaces did). The
8876    /// `Final vote` heading stays first.
8877    fn vote_prompt_full(
8878        &self,
8879        j: usize,
8880        labels: &[char],
8881        language: &str,
8882        candidates: &[Candidate],
8883        base_short: &str,
8884    ) -> String {
8885        let mut text = format!(
8886            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8887            prompt::final_vote(labels, language),
8888            self.state.instruction,
8889            self.candidate_block(candidates, base_short)
8890        );
8891        if let Some(own) = self
8892            .state
8893            .judgements
8894            .get(j)
8895            .filter(|r| !r.ranking.is_empty())
8896        {
8897            let reasons = own
8898                .reasons
8899                .iter()
8900                .map(|(k, v)| format!("- {k}: {v}"))
8901                .collect::<Vec<_>>()
8902                .join("\n");
8903            text.push_str(&format!(
8904                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8905                own.ranking.iter().collect::<String>(),
8906                if reasons.is_empty() {
8907                    ""
8908                } else {
8909                    ", because:\n"
8910                }
8911            ));
8912        }
8913        if !self.state.deliberation.is_empty() {
8914            text.push_str("\n# What was argued before this vote\n");
8915            for t in self.transcript(&[], j) {
8916                text.push_str(&format!(
8917                    "\n## {}{}\n\n{}\n",
8918                    t.who,
8919                    if t.is_self { " (you)" } else { "" },
8920                    t.body.trim()
8921                ));
8922            }
8923        }
8924        text
8925    }
8926
8927    /// Anonymised transcript for judge `self_idx`.
8928    ///
8929    /// The initial rankings are always the opening statements. Seeding them
8930    /// only when no turn had been taken yet meant every judge after the first
8931    /// argued against a single voice instead of against the actual split — the
8932    /// disagreement is the information, so it is always on the table.
8933    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8934        let mut turns = Vec::new();
8935        for j in &self.state.judgements {
8936            if j.ranking.is_empty() {
8937                continue;
8938            }
8939            let reasons = j
8940                .reasons
8941                .iter()
8942                .map(|(k, v)| format!("- {k}: {v}"))
8943                .collect::<Vec<_>>()
8944                .join("\n");
8945            turns.push(Turn {
8946                who: format!("Judge {} (opening ranking)", j.judge),
8947                is_self: j.judge == self_idx + 1,
8948                body: format!(
8949                    "Ranked {}{}{reasons}",
8950                    j.ranking.iter().collect::<String>(),
8951                    if reasons.is_empty() {
8952                        ""
8953                    } else {
8954                        ", because:\n"
8955                    }
8956                ),
8957            });
8958        }
8959        for t in self
8960            .state
8961            .deliberation
8962            .iter()
8963            .flat_map(|r| r.turns.iter())
8964            .chain(current)
8965        {
8966            turns.push(Turn {
8967                who: format!("Judge {}", t.judge),
8968                is_self: t.judge == self_idx + 1,
8969                body: t.body.clone(),
8970            });
8971        }
8972        turns
8973    }
8974}
8975
8976/// Does this seat still hold the context a follow-up prompt would rely on?
8977fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8978    agent::has_session(spec.kind, seat, sessions)
8979}
8980
8981/// The next entry in `roster` after `start`, never wrapping back to the
8982/// front, whose id is not in `tried` yet.
8983///
8984/// Starts one past `start` rather than at the front of `roster`: `start` is
8985/// the seat's own original position, and a seat whose candidate slot already
8986/// sits on the roster's second entry must fall through to the third next, not
8987/// restart at the first — which is very likely a different candidate's own
8988/// agent already. Never wraps back past `start`, for the same reason: an
8989/// entry earlier in the roster than the seat's own position is almost
8990/// certainly some *other* candidate slot's own agent, and once the tail of
8991/// the roster is exhausted there are no more untried agents for *this* seat
8992/// to fall through to — the caller's fallback chain ends there, exactly as
8993/// "no further untried agents remain in the list for that seat" asks for.
8994///
8995/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8996/// the same id twice (an operator's `roles.implementers` typo, or a
8997/// `[[agents]]` list reused across roles) must not let
8998/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8999/// over `roster` either finds an untried id or runs out, so this always
9000/// terminates regardless of duplicates.
9001fn next_untried_in_roster<'a>(
9002    roster: &'a [AgentSpec],
9003    start: usize,
9004    tried: &BTreeSet<String>,
9005) -> Option<&'a AgentSpec> {
9006    roster
9007        .get(start + 1..)?
9008        .iter()
9009        .find(|s| !tried.contains(&s.id))
9010}
9011
9012/// The successor for a seat, shared by all four seat kinds (implement, judge,
9013/// review, advise). `others` holds the ids that *currently* occupy the other
9014/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9015/// sees them), so roster entries beyond the seat count act as spares: a failed
9016/// seat goes to an agent no other seat holds whenever the roster permits.
9017///
9018/// `carried` is `Some` only for the review loop's carried failure history.
9019/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9020/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9021/// whole roster: untried and not another seat's occupant; (3) the plain walk
9022/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9023/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9024/// a better outcome than an empty seat, and it keeps the answer to "is there
9025/// a successor at all" exactly what it was before occupants were considered,
9026/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9027/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9028/// carried failure is a cheaper bet than doubling an agent on the panel, and
9029/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9030/// the same round are handled one at a time, so a seat later in the batch
9031/// sees an earlier one's new occupant but not yet its own freed agent.
9032fn pick_successor<'a>(
9033    roster: &'a [AgentSpec],
9034    start: usize,
9035    tried: &BTreeSet<String>,
9036    carried: Option<&BTreeSet<String>>,
9037    others: &BTreeSet<String>,
9038) -> Option<&'a AgentSpec> {
9039    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9040    roster
9041        .get(start + 1..)
9042        .and_then(|tail| {
9043            tail.iter()
9044                .filter(free)
9045                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9046        })
9047        .or_else(|| {
9048            carried
9049                .is_some()
9050                .then(|| roster.iter().find(free))
9051                .flatten()
9052        })
9053        .or_else(|| match carried {
9054            Some(c) => next_for_seat(roster, start, tried, c),
9055            None => next_untried_in_roster(roster, start, tried),
9056        })
9057}
9058
9059/// The next agent for a seat that carries its failure history across rounds
9060/// (the review loop). `round_tried` is this round's own bound and starts
9061/// empty every round; `carried_failed` only decides priority.
9062///
9063/// First: an id walking forward from `start`, never wrapping, that is neither
9064/// tried this round nor failed in an earlier one. Only when that is exhausted
9065/// does it rescue: the first roster id (in roster order, so this one *does*
9066/// look before `start`) not yet tried this round, which is by then a carried
9067/// failure. Each id is rescued at most once per round, so it cannot loop.
9068fn next_for_seat<'a>(
9069    roster: &'a [AgentSpec],
9070    start: usize,
9071    round_tried: &BTreeSet<String>,
9072    carried_failed: &BTreeSet<String>,
9073) -> Option<&'a AgentSpec> {
9074    roster
9075        .get(start + 1..)?
9076        .iter()
9077        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9078        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9079}
9080
9081/// Where a reviewer seat starts a round: the agent that last answered it when
9082/// it is still on the roster and not marked failed, else the spec's own agent
9083/// unless it failed, else the next roster agent that has not failed, else the
9084/// spec's own agent again (the whole roster failed). Ids no longer on the
9085/// roster are ignored. An empty roster has no handover, so the spec stands.
9086fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9087    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9088        return spec;
9089    };
9090    let ok = |id: &str| !h.failed.contains(id);
9091    if let Some(last) = h.last_ok.as_deref()
9092        && ok(last)
9093        && let Some(s) = roster.iter().find(|s| s.id == last)
9094    {
9095        return s.clone();
9096    }
9097    if ok(&spec.id) {
9098        return spec;
9099    }
9100    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9101    roster
9102        .get(from + 1..)
9103        .into_iter()
9104        .flatten()
9105        .chain(roster.iter())
9106        .find(|s| ok(&s.id))
9107        .cloned()
9108        .unwrap_or(spec)
9109}
9110
9111/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9112/// seed so a CLI that mints its session id up front (`--session-id`) never
9113/// reuses the uuid the previous agent already opened under the same seat key.
9114pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9115    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9116}
9117
9118/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9119/// so the classifier below and the code that builds the message cannot drift.
9120const TIMED_OUT: &str = "timed out";
9121
9122impl FailClass {
9123    /// `None` for an answer; otherwise how the turn failed.
9124    fn of(out: &AgentOutcome) -> Option<Self> {
9125        match out {
9126            AgentOutcome::Ok(_) => None,
9127            AgentOutcome::Quota(_) => Some(Self::Quota),
9128            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9129            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9130            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9131            // Not a failure of the agent. Callers check for it first; `None`
9132            // here means "no class to hand over on".
9133            AgentOutcome::ResourceBlocked(_) => None,
9134        }
9135    }
9136
9137    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9138    fn stem_word(&self) -> &'static str {
9139        match self {
9140            Self::Quota => "quota",
9141            _ => "handover",
9142        }
9143    }
9144}
9145
9146/// The message's first line with its variable parts removed — digit runs and
9147/// path-like tokens — so "exit code 2" and "exit code 7" read
9148/// as one kind of failure.
9149fn failure_signature(msg: &str) -> String {
9150    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9151    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9152    // the historical machine key while the process diagnosis uses readable prose.
9153    if let Some(code) = line
9154        .strip_prefix("process exited with no usable output (exit code ")
9155        .and_then(|s| s.strip_suffix(')'))
9156        .and_then(|s| s.parse::<i32>().ok())
9157    {
9158        line = format!("exited with some({code}) and no usable output");
9159    } else if matches!(
9160        line.as_str(),
9161        "process exited with no usable output (no exit code reported)"
9162            | "process exited with no usable output (exit code unavailable)"
9163    ) {
9164        line = "exited with none and no usable output".to_owned();
9165    }
9166    let mut out = Vec::new();
9167    for word in line.split_whitespace() {
9168        if word.contains('/') || word.contains('\\') {
9169            out.push("<path>".to_owned());
9170            continue;
9171        }
9172        let mut w = String::new();
9173        let mut in_digits = false;
9174        for c in word.chars() {
9175            if c.is_ascii_digit() {
9176                if !in_digits {
9177                    w.push('#');
9178                }
9179                in_digits = true;
9180            } else {
9181                in_digits = false;
9182                w.push(c);
9183            }
9184        }
9185        out.push(w);
9186    }
9187    out.join(" ").chars().take(120).collect()
9188}
9189
9190/// Whether a seat that just failed with `cur` may go to the next roster agent.
9191/// A quota or a timeout always may. Any other failure may not when the agent
9192/// before it failed the same way: an error the prompt causes would otherwise
9193/// walk the whole roster. `prev` is the class of the immediately preceding
9194/// agent's failure, so a quota or timeout in between breaks the run of
9195/// identical failures by itself.
9196fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9197    match cur {
9198        FailClass::Quota | FailClass::Timeout => true,
9199        FailClass::Other(_) => prev != Some(cur),
9200    }
9201}
9202
9203/// A short human reason for a failed outcome, for the handover record.
9204fn fail_reason(out: &AgentOutcome) -> String {
9205    match out {
9206        AgentOutcome::Ok(_) => String::new(),
9207        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9208        AgentOutcome::Dropped(o) => format!(
9209            "the CLI dropped the stream ({})",
9210            o.dropped
9211                .as_ref()
9212                .map(|d| d.why.as_str())
9213                .unwrap_or("it ended without delivering its answer")
9214        ),
9215        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9216            e.lines().next().unwrap_or("").chars().take(160).collect()
9217        }
9218    }
9219}
9220
9221/// Note one handover in the run: the structured record and, in the timeline,
9222/// the sentence a person reads. A quota keeps the wording it always had.
9223pub(crate) fn record_handover(
9224    state: &mut RunState,
9225    node: &str,
9226    seat: &str,
9227    from: &str,
9228    to: &str,
9229    class: &FailClass,
9230    reason: &str,
9231) {
9232    let message = if *class == FailClass::Quota {
9233        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9234    } else {
9235        format!("{seat}: handed over {from} -> {to} ({reason})")
9236    };
9237    state.event(node, message);
9238    state.handovers.push(Handover {
9239        at: Timestamp::now(),
9240        node: node.to_owned(),
9241        seat: seat.to_owned(),
9242        from: from.to_owned(),
9243        to: to.to_owned(),
9244        reason: reason.to_owned(),
9245    });
9246}
9247
9248/// Did this reply report running a command whose own CLI never confirmed an
9249/// exit status?
9250///
9251/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9252/// command *finished* (see that type's own doc), so this can only be `true`
9253/// for a command whose completion event carried no readable exit code — not
9254/// for one that simply is not mentioned at all. That is the one signal this
9255/// crate can read, from the same record `state.jobs` renders, about a reply
9256/// standing next to work its own CLI cannot vouch for finishing; it is
9257/// deliberately not a check on the exit code's *value* (a fixer legitimately
9258/// runs a command that fails mid-iteration before it succeeds) and not a
9259/// guess at a command still running in the background (which emits no event
9260/// at all, and so leaves no evidence here to find).
9261fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9262    commands.iter().any(|c| c.exit_code.is_none())
9263}
9264
9265/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9266/// trusted as a verified no-op — the adoption guard's own text-level half.
9267///
9268/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9269/// timed out): a marker only earns the benefit of the doubt from a turn the
9270/// CLI itself vouches for finishing properly, the same house style
9271/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9272/// *fix* report to for `commands`. A candidate that timed out, exited
9273/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9274/// is, whatever prose it wrote — this returns `None` before it ever looks at
9275/// `text`. The remaining guards (the tree really is empty, the evidence is
9276/// non-empty) are the caller's: this only reads what the reply *claimed*.
9277fn verified_noop_claim(
9278    usable: bool,
9279    commands: &[agent::CommandEvidence],
9280    text: &str,
9281) -> Option<String> {
9282    (usable && !has_unconfirmed_command(commands))
9283        .then(|| verdict::verified_noop(text))
9284        .flatten()
9285}
9286
9287fn short(commit: &str) -> String {
9288    commit.chars().take(7).collect()
9289}
9290
9291fn make_executable(path: &Path) -> Result<()> {
9292    #[cfg(unix)]
9293    {
9294        use std::os::unix::fs::PermissionsExt as _;
9295        let mut perms = std::fs::metadata(path)?.permissions();
9296        perms.set_mode(0o755);
9297        std::fs::set_permissions(path, perms)?;
9298    }
9299    #[cfg(not(unix))]
9300    {
9301        let _ = path;
9302    }
9303    Ok(())
9304}
9305
9306/// What every seat in one batch shares: where the answers are attributed, the
9307/// prompt overlay they inherit, and the build cache they are told to use.
9308///
9309/// A struct rather than four more parameters: `wave` also needs the run's
9310/// state (to record who is answering right now) and the attempt number, and
9311/// eight positional arguments is both unreadable and a clippy error.
9312struct WaveCtx<'a> {
9313    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9314    /// paid for it.
9315    run: &'a str,
9316    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9317    node: &'a str,
9318    prompts: &'a Prompts,
9319    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9320    cache: Option<&'a Path>,
9321    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9322    /// `JobRecord::round`. `None` for every other node.
9323    round: Option<usize>,
9324    /// Carry each seat's failed-agent history across waves (the review loop
9325    /// only): start-of-round priority and handover choice read
9326    /// [`RunState::seat_history`], and every answer or failure writes it.
9327    carry_seats: bool,
9328    /// The run's park / interrupt requests, so a wait for a shared resource
9329    /// can be abandoned the moment the operator asks to park.
9330    pauses: Option<(&'a Pause, &'a Pause)>,
9331}
9332
9333/// Shared scheduling policy for operations and spawned seat invocations.
9334fn repository_wait(
9335    state: &RunState,
9336    pauses: Option<(&Pause, &Pause)>,
9337) -> crate::repo_lock::WaitPolicy {
9338    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9339    let inherited = crate::repo_lock::WaitPolicy::current();
9340    crate::repo_lock::WaitPolicy::new(
9341        Duration::from_secs(state.config.graph.timeout_resource_wait),
9342        move || {
9343            inherited.cancelled()
9344                || pauses
9345                    .as_ref()
9346                    .is_some_and(|(a, b)| a.parked() || b.parked())
9347        },
9348    )
9349}
9350
9351fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9352    if error.is::<crate::repo_lock::ResourceBlocked>() {
9353        state.resource_block = Some(crate::run::ResourceBlock {
9354            cause: crate::run::ResourceBlockCause::Other,
9355            node: "repository".into(),
9356            kind: if error.to_string().contains("disk pressure") {
9357                "disk pressure"
9358            } else {
9359                "repository coordination"
9360            }
9361            .into(),
9362            detail: error.to_string(),
9363            at: Timestamp::now(),
9364        });
9365        state.event("resource", error.to_string());
9366    }
9367}
9368
9369async fn coordinate_operation(
9370    state: &mut RunState,
9371    cwd: &Path,
9372    wait: &crate::repo_lock::WaitPolicy,
9373) -> Result<Option<crate::repo_lock::SeatLease>> {
9374    let result =
9375        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9376            .await;
9377    if let Err(e) = &result {
9378        record_repository_block(state, e);
9379    }
9380    result
9381}
9382
9383/// Run one job, honouring the parallelism budget.
9384async fn run_one(
9385    job: SeatJob,
9386    sem: Arc<Semaphore>,
9387    ctx: &WaveCtx<'_>,
9388    state: &mut RunState,
9389    attempt: usize,
9390) -> (SeatState, AgentOutcome) {
9391    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9392        .await
9393        .pop()
9394        .expect("one job in, one result out");
9395    (seat, out)
9396}
9397
9398/// Run every job concurrently, capped by the semaphore, preserving order.
9399///
9400/// Every seat in the batch is recorded into [`RunState::active`] before the
9401/// wave starts and cleared as each answer lands, so the run's own record says
9402/// who is still being waited on rather than only who finished.
9403async fn wave(
9404    jobs: Vec<SeatJob>,
9405    sem: Arc<Semaphore>,
9406    ctx: &WaveCtx<'_>,
9407    state: &mut RunState,
9408    attempt: usize,
9409) -> Vec<(usize, SeatState, AgentOutcome)> {
9410    let WaveCtx {
9411        run,
9412        node,
9413        prompts,
9414        cache,
9415        round,
9416        carry_seats: _,
9417        pauses,
9418    } = *ctx;
9419    for job in &jobs {
9420        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9421    }
9422    if let Err(e) = state.save() {
9423        // A failed persist of "who is answering right now" must not abort the
9424        // wave: the seats are already being asked, and the alternative is
9425        // losing the answers to save a status line nobody may even be
9426        // watching.
9427        tracing::warn!("could not persist in-progress seats: {e:#}");
9428    }
9429    // Hold the shared build cache's lease for the whole batch, not per job:
9430    // several candidates (an implement wave) or a fixer legitimately share
9431    // one cache concurrently within this run, and that stays untouched — a
9432    // single lease taken once for the whole wave and released once it is
9433    // done is what stops a *different* borrower (another run's own wave, its
9434    // e2e/gate, a human's `magi review`) from interleaving a build into the
9435    // same directory while this one is in flight. Best-effort, not
9436    // all-or-nothing: a wave that cannot get the lease within its own
9437    // longest job's budget still runs — an hour of paid implementer calls is
9438    // not thrown away over cache contention — but every write-allowed seat
9439    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9440    // below), the same fallback a read-only seat always gets, rather than
9441    // building into a directory this run was never granted. The identity
9442    // record is still invalidated below either way, so the next tracked
9443    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9444    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9445    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9446    // never borrowed from a job's agent budget (and never charged to it either,
9447    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9448    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9449    let cache_guard = if let Some(cache_dir) = cache {
9450        if jobs_had_a_writer {
9451            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9452            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9453            match acquire_cache_lease(
9454                state,
9455                cache_dir,
9456                &owner,
9457                resource_limit,
9458                node,
9459                node,
9460                &cancel,
9461            )
9462            .await
9463            {
9464                Ok(Some(g)) => Some(g),
9465                // Parking: leave without spawning anything. The run parks at
9466                // the next boundary and the seats are asked afresh on resume.
9467                Ok(None) => {
9468                    if let Some(block) = &mut state.resource_block {
9469                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9470                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9471                        };
9472                    }
9473                    return block_wave(
9474                        jobs,
9475                        state,
9476                        node,
9477                        attempt,
9478                        "parked while waiting for the shared build cache",
9479                    );
9480                }
9481                // An unreadable lease is not contention: keep the old
9482                // fallback (no shared cache for this wave's seats).
9483                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9484                Err(e) => {
9485                    // Writing seats may not build into a cache this run does
9486                    // not hold, and an agent that cannot be given one is not
9487                    // asked at all: the run stops resumably instead.
9488                    let why = format!("{e:#}");
9489                    state.event(
9490                        "resource",
9491                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9492                    );
9493                    return block_wave(jobs, state, node, attempt, &why);
9494                }
9495            }
9496        } else {
9497            None
9498        }
9499    } else {
9500        None
9501    };
9502    // Every job keeps its whole budget: the wait above was not the agent's.
9503    // Restart the seats' clocks so "started" means "being spawned now".
9504    state.seats_spawning(node, attempt);
9505    if let Err(e) = state.save() {
9506        tracing::warn!("could not persist the start of a wave: {e:#}");
9507    }
9508    // Queueing behind this wave's own seats for a parallelism slot is normal
9509    // (each of them is bounded by its own agent timeout); only a wait beyond
9510    // all of them plus the resource limit means the slots never came back.
9511    let slot_limit = resource_limit
9512        + jobs
9513            .iter()
9514            .map(|j| j.timeout)
9515            .max()
9516            .unwrap_or_default()
9517            .saturating_mul(jobs.len() as u32);
9518    let mut set = tokio::task::JoinSet::new();
9519    // Seats announce themselves here the moment they hold a parallelism
9520    // permit, i.e. just before their CLI is spawned.
9521    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9522    let overlay = prompts.overlay(node);
9523    let repository_wait = repository_wait(state, pauses);
9524    for (i, mut job) in jobs.into_iter().enumerate() {
9525        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9526        if cache.is_some() {
9527            job.prompt.push('\n');
9528            job.prompt
9529                .push_str(&prompt::build_cache_note(node, job.allow_write));
9530        }
9531        let sem = Arc::clone(&sem);
9532        let run = run.to_owned();
9533        let node = node.to_owned();
9534        // Only implementers were told about the task's attachments, so only
9535        // their seats get the directory widened for reading.
9536        let attachments = if node == "implement" {
9537            state.attachments.clone()
9538        } else {
9539            Vec::new()
9540        };
9541        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9542        // `prompt::build_cache_note`'s doc for why setting it anyway is
9543        // exactly how a sandboxed reviewer's write refusal got reported as a
9544        // defect in the patch, not a property of its own seat. And a
9545        // write-allowed one is handed it only when the lease above was
9546        // actually acquired: a wave that could not get it (`cache_guard` is
9547        // `None`, see its own comment) must not send seats to build into a
9548        // directory this run does not hold - that is the exact concurrent,
9549        // unmanaged-write race this module exists to prevent, not something
9550        // "proceeding anyway" is allowed to reintroduce.
9551        let cache = cache
9552            .filter(|_| job.allow_write && cache_guard.is_some())
9553            .map(Path::to_path_buf);
9554        let spawned_tx = spawned_tx.clone();
9555        let repository_wait = repository_wait.clone();
9556        let coordination = state.config.coordination.clone();
9557        let editing_network = state.config.graph.editing_network;
9558        let fetched_base = state.fetched_base.clone();
9559        set.spawn(async move {
9560            // A wait for a parallelism slot is scheduling, not the agent: it
9561            // is bounded by its own limit and ends as a resource block, never
9562            // as a timeout of the agent that was not started.
9563            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9564                Ok(p) => p,
9565                Err(_) => {
9566                    return (
9567                        i,
9568                        job.seat,
9569                        AgentOutcome::ResourceBlocked(format!(
9570                            "no parallel slot became free within {}s",
9571                            slot_limit.as_secs()
9572                        )),
9573                        false,
9574                    );
9575                }
9576            };
9577            spawned_tx.send(job.seat.key.clone()).ok();
9578            let mut seat = job.seat;
9579            let out = repository_wait
9580                .scope(agent::invoke(
9581                    &job.spec,
9582                    &mut seat,
9583                    &Invocation {
9584                        coordination: coordination.as_ref(),
9585                        editing_network,
9586                        fetched_base: fetched_base.as_ref(),
9587                        cwd: &job.cwd,
9588                        prompt: &job.prompt,
9589                        timeout: job.timeout,
9590                        allow_write: job.allow_write,
9591                        unsandboxed: false,
9592                        sessions: job.sessions,
9593                        artifacts: &job.artifacts,
9594                        stem: &job.stem,
9595                        run: &run,
9596                        node: &node,
9597                        cache_dir: cache.as_deref(),
9598                        attachments: &attachments,
9599                        writable: &[],
9600                    },
9601                ))
9602                .await;
9603            let interrupted = out
9604                .as_ref()
9605                .err()
9606                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9607            let out = match out {
9608                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9609                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9610                // Billed work the CLI failed to hand over is not an ordinary
9611                // failure, but its text is the CLI's raw error JSON, not an
9612                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9613                // read it as one by forgetting to check. `usable()` is always
9614                // false here (dropped implies an empty response), so this has
9615                // to be checked before the catch-all `Failed` below or the
9616                // one shape this exists for is lost with the rest.
9617                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9618                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9619                Ok(o) => AgentOutcome::Failed(format!(
9620                    "process exited with no usable output ({})",
9621                    crate::notices::exit_status(o.exit_code)
9622                )),
9623                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9624                    AgentOutcome::ResourceBlocked(e.to_string())
9625                }
9626                Err(e) => AgentOutcome::Failed(e.to_string()),
9627            };
9628            (i, seat, out, interrupted)
9629        });
9630    }
9631    drop(spawned_tx);
9632    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9633    loop {
9634        let joined = tokio::select! {
9635            Some(key) = spawned_rx.recv() => {
9636                // The clock of this seat starts now, not when it queued.
9637                state.seat_spawned(&key);
9638                if let Err(e) = state.save() {
9639                    tracing::warn!("could not persist a seat's start: {e:#}");
9640                }
9641                continue;
9642            }
9643            joined = set.join_next() => joined,
9644        };
9645        let Some(joined) = joined else {
9646            break;
9647        };
9648        let (i, seat, out, interrupted) = match joined {
9649            Ok(v) => v,
9650            // No seat to clear: a panicked task never reported which one it
9651            // was. The defensive sweep below this loop is what stops that
9652            // seat's `active` entry from surviving forever.
9653            Err(e) => {
9654                tracing::error!("agent task panicked: {e}");
9655                continue;
9656            }
9657        };
9658        state.seat_finished(&seat.key);
9659        if interrupted
9660            && node == "implement"
9661            && let Some(label) = state
9662                .candidates
9663                .iter()
9664                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9665                .map(|candidate| candidate.label)
9666        {
9667            state.candidate_preservation.insert(
9668                label,
9669                "interrupted implementation awaiting persistence".into(),
9670            );
9671            state.interrupted_implementations.insert(label);
9672        }
9673        if let AgentOutcome::ResourceBlocked(why) = &out
9674            && state.resource_block.is_none()
9675        {
9676            state.resource_block = Some(crate::run::ResourceBlock {
9677                cause: crate::run::ResourceBlockCause::Other,
9678                node: node.to_owned(),
9679                kind: if why.contains("disk pressure") {
9680                    "disk pressure"
9681                } else if why.contains("repository coordination") {
9682                    "repository coordination"
9683                } else {
9684                    "parallel slot"
9685                }
9686                .to_owned(),
9687                detail: why.clone(),
9688                at: Timestamp::now(),
9689            });
9690            let activity = if interrupted {
9691                "agent execution interrupted"
9692            } else {
9693                "no agent was started"
9694            };
9695            state.event("resource", format!("{node}: {activity} - {why}"));
9696        }
9697        record_jobs(state, node, round, &seat.key, &out);
9698        if let Err(e) = state.save() {
9699            tracing::warn!("could not persist a seat's completion: {e:#}");
9700        }
9701        if collected.len() <= i {
9702            collected.resize_with(i + 1, || None);
9703        }
9704        collected[i] = Some((i, seat, out));
9705    }
9706    // Belt-and-braces for the panic branch above: every seat this exact batch
9707    // started shares this `(node, attempt)` pair, and every seat that finished
9708    // normally already cleared itself, so anything left tagged with it here
9709    // can only be a panicked task's leftover. Cleared unconditionally rather
9710    // than left to read as still answering forever.
9711    if state
9712        .active
9713        .values()
9714        .any(|a| a.node == node && a.attempt == attempt)
9715    {
9716        state
9717            .active
9718            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9719        if let Err(e) = state.save() {
9720            tracing::warn!("could not persist the end of a wave: {e:#}");
9721        }
9722    }
9723    // Whether or not the lease above was actually held, several worktrees
9724    // may just have built into the cache with nothing here able to name one
9725    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9726    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9727    // clean it might not have strictly needed; trusting a stale match would
9728    // cost it a wrong answer.
9729    if let Some(cache_dir) = cache
9730        && jobs_had_a_writer
9731    {
9732        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9733    }
9734    if let Some(guard) = cache_guard {
9735        guard.release();
9736    }
9737    collected.into_iter().flatten().collect()
9738}
9739
9740/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9741/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9742/// this is data collection, not the fix-specific completion contract in
9743/// [`Runner::continue_fix_report`], and applies regardless of which node
9744/// asked.
9745///
9746/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9747/// evidence from; `Failed` does not, and correctly contributes nothing — a
9748/// timeout or crash is not itself evidence about a command the seat may have
9749/// started.
9750fn record_jobs(
9751    state: &mut RunState,
9752    node: &str,
9753    round: Option<usize>,
9754    seat: &str,
9755    out: &AgentOutcome,
9756) {
9757    let commands: &[agent::CommandEvidence] = match out {
9758        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9759        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9760    };
9761    let checked_at = Timestamp::now();
9762    for c in commands {
9763        state.jobs.push(JobRecord {
9764            node: node.to_owned(),
9765            round,
9766            seat: seat.to_owned(),
9767            id: c.id.clone(),
9768            description: c.description.clone(),
9769            checked_at,
9770            status: match c.exit_code {
9771                Some(0) => JobStatus::Completed,
9772                Some(_) => JobStatus::Failed,
9773                None => JobStatus::Unknown,
9774            },
9775            exit_code: c.exit_code,
9776            result_summary: c.result_summary.clone(),
9777            source: c.source.clone(),
9778        });
9779    }
9780}
9781
9782/// Is a review round clean, given how many reviewer seats answered against
9783/// how many the round expected?
9784///
9785/// A seat that never answered (timeout, crash, unparsable output) is not a
9786/// seat that read the patch and found nothing — treating it as such is
9787/// exactly the bug this function exists to close. Under the default `block`
9788/// policy a missing seat can never be clean; `warn` still requires the seats
9789/// that *did* answer to have found nothing blocking and verification to be
9790/// green.
9791///
9792/// `quota_missing` narrows that `block` default for exactly one cause of
9793/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9794/// a session limit lifts by the very next round buys nothing — the seat is
9795/// asked again with the same quota — so once every missing seat is accounted
9796/// for by a quota loss (and at least one seat *did* answer, so a decision has
9797/// something to rest on) the round is decided on the panel that could answer,
9798/// same as `warn` would. A panel that lost every seat to quota is not
9799/// decided here: `answered == 0` falls through to the existing `block`
9800/// fallback so a fully collapsed panel still waits rather than landing on no
9801/// review at all.
9802fn round_is_clean(
9803    blocking: usize,
9804    e2e_ok: bool,
9805    answered: usize,
9806    expected: usize,
9807    quota_missing: usize,
9808    policy: IncompleteReviewPolicy,
9809) -> bool {
9810    if blocking != 0 || !e2e_ok {
9811        return false;
9812    }
9813    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9814        return true;
9815    }
9816    answered > 0 && expected - answered <= quota_missing
9817}
9818
9819/// The review loop's own conclusion, derived entirely from its persisted
9820/// round records and the round budget that produced them — never from
9821/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9822/// recomputes the identical answer regardless of what an earlier node in the
9823/// same walk, or a previous walk, did to `status`.
9824///
9825/// `None` while more rounds remain to try, including when review never ran
9826/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9827/// gone clean, or the budget is spent, or the tree has stopped moving (see
9828/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9829///
9830/// - An incomplete panel that raised nothing is missing input, not a
9831///   verified tree — never a hand-off candidate, whatever verification said
9832///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9833/// - Otherwise, green e2e on the last round hands off (see
9834///   [`Runner::stop_reviewing`]); red e2e blocks.
9835///
9836/// A last round whose own verification is still `ResourceBlocked` — magi
9837/// itself never got a command to run, not evidence the patch is broken —
9838/// is neither: this returns `None` for it too, the same as "more rounds
9839/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9840/// handling of that shape) instead of this cheap recomputation guessing a
9841/// verdict a real attempt never produced.
9842pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9843    if max_rounds == 0
9844        || reviews.last().is_some_and(|r| {
9845            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9846        })
9847    {
9848        return Some(RunStatus::Gating);
9849    }
9850    let last = reviews.last()?;
9851    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9852    if reviews.len() < max_rounds && !stagnant {
9853        return None;
9854    }
9855    if last.incomplete() && last.blocking == 0 {
9856        return Some(RunStatus::Blocked);
9857    }
9858    if last.e2e_status() == E2eStatus::ResourceBlocked
9859        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9860    {
9861        return None;
9862    }
9863    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9864        RunStatus::Gating
9865    } else {
9866        RunStatus::Blocked
9867    })
9868}
9869
9870/// How long a re-ask may take, given the budget the first attempt had.
9871///
9872/// A `nudged` retry is a request to restate an answer the seat has already
9873/// worked out: it carries no new work, so it does not deserve the original
9874/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9875/// 133 seconds while a third sat for over ten minutes on a resumed session
9876/// holding 410 KB of prior output - and because the retry had inherited the
9877/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9878/// judging round whose other seats were long finished.
9879///
9880/// A quarter of the budget, with a floor so that a deliberately short timeout
9881/// does not collapse to nothing. A retry that re-sends the whole prompt
9882/// (because the seat kept no context) is the original job again, and keeps the
9883/// original budget.
9884pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9885    if nudged {
9886        (full / 4).max(Duration::from_secs(120)).min(full)
9887    } else {
9888        full
9889    }
9890}
9891
9892/// Run a wave and parse each reply, re-asking the seats whose reply was
9893/// unusable.
9894///
9895/// The re-ask is a nudge rather than the whole prompt again when the seat still
9896/// holds its conversation, which is the difference between a cheap retry and
9897/// paying for the entire candidate set twice.
9898///
9899/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9900/// successor in `roster` is handed to it instead of being re-asked: the
9901/// handover is the retry. A seat with no successor left (a single-agent
9902/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9903/// the asks to one seat in one node number at most
9904/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9905/// is asked once (a dropped stream is nudged first), and only the last agent
9906/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9907/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9908/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9909/// other failure stops the chain when the previous agent failed the same way
9910/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9911/// [`has_context`] is false and the job's own full prompt and full budget are
9912/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9913/// intermediate ones are not losses) and is returned as a failure like any
9914/// other absent seat — the caller decides whether the panel still has a
9915/// quorum. An empty `roster` disables handover: failures are nudged as they
9916/// always were, and a quota is simply lost. A reply that fails to parse or
9917/// validate is the prompt's doing and is only ever nudged, never handed over.
9918///
9919/// The returned [`SeatState`] names the agent that answered (or tried last).
9920#[allow(clippy::too_many_arguments)]
9921async fn ask_json_wave<T>(
9922    jobs: Vec<SeatJob>,
9923    sem: Arc<Semaphore>,
9924    retries: usize,
9925    roster: &[AgentSpec],
9926    ctx: &WaveCtx<'_>,
9927    losses: &mut Vec<QuotaLoss>,
9928    state: &mut RunState,
9929    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9930) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9931where
9932    T: serde::de::DeserializeOwned + Send + 'static,
9933{
9934    ask_wave_with(
9935        jobs,
9936        sem,
9937        retries,
9938        roster,
9939        ctx,
9940        losses,
9941        state,
9942        &|text: &str| {
9943            let v = verdict::extract_json::<T>(text)?;
9944            validate(&v)?;
9945            Ok(v)
9946        },
9947    )
9948    .await
9949}
9950
9951/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9952/// a node whose answer is prose (deliberation) shares the same handover,
9953/// failure classification, quota bookkeeping and bounds instead of a copy.
9954///
9955/// A seat handed to another roster agent is sent the job's `handover` prompt
9956/// (when it has one) rather than `prompt`: the new agent has no session, so a
9957/// resume-style prompt would be incomplete. That holds for the handover ask
9958/// and for every nudge to that agent whose `has_context` is false.
9959#[allow(clippy::too_many_arguments)]
9960async fn ask_wave_with<T>(
9961    jobs: Vec<SeatJob>,
9962    sem: Arc<Semaphore>,
9963    retries: usize,
9964    roster: &[AgentSpec],
9965    ctx: &WaveCtx<'_>,
9966    losses: &mut Vec<QuotaLoss>,
9967    state: &mut RunState,
9968    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9969) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9970where
9971    T: Send + 'static,
9972{
9973    let n = jobs.len();
9974    let originals: Vec<SeatJob> = jobs;
9975    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9976    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9977    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9978    // answer, N once it has gone through N nudges. Read back once this
9979    // returns, so a caller building a history record (`ReviewRecord`) can
9980    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9981    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9982    // field's own doc.
9983    let mut nudges: Vec<usize> = vec![0; n];
9984    // The agent now occupying each seat, the ids it has already been through,
9985    // where in the roster the walk began, the class of the last failure, and
9986    // the stem word of a handover not yet asked (full prompt, full budget).
9987    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9988    let mut tried: Vec<BTreeSet<String>> = specs
9989        .iter()
9990        .map(|s| BTreeSet::from([s.id.clone()]))
9991        .collect();
9992    let starts: Vec<usize> = specs
9993        .iter()
9994        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9995        .collect();
9996    let carry = ctx.carry_seats && !roster.is_empty();
9997    // Carried across rounds: ids that failed the seat earlier, and how the
9998    // last failure went (so a repeat of it is not handed over again).
9999    let carried: Vec<BTreeSet<String>> = originals
10000        .iter()
10001        .map(|j| {
10002            state
10003                .seat_history
10004                .get(&j.seat.key)
10005                .filter(|_| carry)
10006                .map(|h| h.failed.clone())
10007                .unwrap_or_default()
10008        })
10009        .collect();
10010    let mut prev: Vec<Option<FailClass>> = originals
10011        .iter()
10012        .map(|j| {
10013            state
10014                .seat_history
10015                .get(&j.seat.key)
10016                .filter(|_| carry)
10017                .and_then(|h| h.last_fail.clone())
10018        })
10019        .collect();
10020    let next_agent =
10021        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10022            let others: BTreeSet<String> = specs
10023                .iter()
10024                .enumerate()
10025                .filter(|(j, _)| *j != i)
10026                .map(|(_, s)| s.id.clone())
10027                .collect();
10028            pick_successor(
10029                roster,
10030                starts[i],
10031                tried,
10032                carry.then(|| &carried[i]),
10033                &others,
10034            )
10035            .cloned()
10036        };
10037    let mut fresh: Vec<Option<String>> = vec![None; n];
10038    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10039    let mut pending: Vec<usize> = (0..n).collect();
10040
10041    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10042    // asks: an agent with a successor is asked once and handed over, and only
10043    // a seat with no successor spends `retries` nudges on the same agent. This
10044    // only guarantees the loop's own termination whatever those say.
10045    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10046    for round in 0..max_rounds {
10047        if pending.is_empty() {
10048            break;
10049        }
10050        let mut batch = Vec::with_capacity(pending.len());
10051        let mut renudged: Vec<&str> = Vec::new();
10052        for &i in &pending {
10053            let src = &originals[i];
10054            // A seat now held by another agent than the job named has no
10055            // session of its own: it gets the full-context prompt whenever
10056            // it is asked in full (the handover ask, a nudge it cannot
10057            // resume).
10058            let full: &str = match &src.handover {
10059                Some(h) if specs[i].id != src.spec.id => h,
10060                _ => &src.prompt,
10061            };
10062            // The prompt and the budget are one decision: a nudge restates
10063            // finished work, a re-sent prompt redoes it.
10064            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10065                (
10066                    full.to_owned(),
10067                    src.timeout,
10068                    format!("{}-{word}-{}", src.stem, specs[i].id),
10069                )
10070            } else if nudges[i] == 0 {
10071                (full.to_owned(), src.timeout, src.stem.clone())
10072            } else {
10073                renudged.push(src.seat.key.as_str());
10074                let why = done[i]
10075                    .as_ref()
10076                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10077                    .unwrap_or_else(|| "no parsable answer".to_owned());
10078                let nudge = prompt::nudge(&why);
10079                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10080                let prompt = if nudged {
10081                    nudge
10082                } else {
10083                    format!("{full}\n\n---\n\n{nudge}")
10084                };
10085                (
10086                    prompt,
10087                    retry_budget(src.timeout, nudged),
10088                    format!("{}-retry{}", src.stem, nudges[i]),
10089                )
10090            };
10091            batch.push(SeatJob {
10092                spec: specs[i].clone(),
10093                seat: seats[i].clone(),
10094                cwd: src.cwd.clone(),
10095                prompt,
10096                timeout,
10097                allow_write: src.allow_write,
10098                sessions: src.sessions,
10099                artifacts: src.artifacts.clone(),
10100                stem,
10101                handover: None,
10102            });
10103        }
10104
10105        if !renudged.is_empty() {
10106            state.event(
10107                ctx.node,
10108                format!("retry {round}: re-asking {}", renudged.join(", ")),
10109            );
10110        }
10111        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10112        let mut still = Vec::new();
10113        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10114            seats[i] = seat;
10115            // No agent was asked: nothing to hand over, nudge, remember or
10116            // charge. The caller sees `resource_block` and stops the node.
10117            if let AgentOutcome::ResourceBlocked(why) = &out {
10118                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10119                continue;
10120            }
10121            let class = FailClass::of(&out);
10122            // A dropped stream is nudged first (the conversation is still
10123            // there to pick up); only a seat whose nudges are spent hands over.
10124            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10125                && nudges[i] < retries
10126                && !roster.is_empty();
10127            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10128                let next = should_hand_over(prev[i].as_ref(), &cur)
10129                    .then(|| next_agent(i, &tried[i], &specs))
10130                    .flatten();
10131                if carry {
10132                    let h = state
10133                        .seat_history
10134                        .entry(originals[i].seat.key.clone())
10135                        .or_default();
10136                    h.failed.insert(specs[i].id.clone());
10137                    h.last_fail = Some(cur.clone());
10138                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10139                        h.last_ok = None;
10140                    }
10141                    // Saved before the next agent is asked, so a restart in
10142                    // between does not forget who failed.
10143                    if let Err(e) = state.save() {
10144                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10145                    }
10146                }
10147                if let Some(next) = next {
10148                    record_handover(
10149                        state,
10150                        ctx.node,
10151                        &originals[i].seat.key,
10152                        &specs[i].id,
10153                        &next.id,
10154                        &cur,
10155                        &fail_reason(&out),
10156                    );
10157                    tried[i].insert(next.id.clone());
10158                    prev[i] = Some(cur.clone());
10159                    seats[i] =
10160                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10161                    specs[i] = next;
10162                    fresh[i] = Some(cur.stem_word().to_owned());
10163                    nudges[i] = 0;
10164                    done[i] = Some(Err(anyhow::anyhow!(
10165                        "handed over after: {}",
10166                        fail_reason(&out)
10167                    )));
10168                    still.push(i);
10169                    continue;
10170                }
10171            }
10172            if carry
10173                && !nudge_first
10174                && let Some(cur) = class.clone()
10175            {
10176                // The chain ended here (no successor, or a repeated failure).
10177                let h = state
10178                    .seat_history
10179                    .entry(originals[i].seat.key.clone())
10180                    .or_default();
10181                h.failed.insert(specs[i].id.clone());
10182                h.last_fail = Some(cur);
10183            }
10184            let parsed = match out {
10185                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10186                AgentOutcome::Quota(o) => {
10187                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10188                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10189                }
10190                // Not a parseable answer: the nudge loop re-asks it, which is
10191                // exactly what a dropped stream needs. Just don't hand its raw
10192                // error JSON to `extract_json`.
10193                AgentOutcome::Dropped(o) => {
10194                    let why = o
10195                        .dropped
10196                        .as_ref()
10197                        .map(|d| d.why.as_str())
10198                        .unwrap_or("the CLI ended the stream without delivering its answer");
10199                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10200                }
10201                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10202                    Err(anyhow::anyhow!(e))
10203                }
10204            };
10205            let quota = class == Some(FailClass::Quota);
10206            let failed = parsed.is_err();
10207            done[i] = Some(parsed);
10208            if carry && !failed {
10209                let h = state
10210                    .seat_history
10211                    .entry(originals[i].seat.key.clone())
10212                    .or_default();
10213                h.failed.remove(&specs[i].id);
10214                h.last_ok = Some(specs[i].id.clone());
10215                h.last_fail = None;
10216            }
10217            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10218            // fail the same way; and never re-ask a seat that already parsed.
10219            // A failed agent that still has a successor is not re-asked
10220            // either: the handover was its remedy and has just been refused
10221            // (the chain stops on a repeated failure class). A seat with no
10222            // successor left (a single-agent roster, the roster's tail, or an
10223            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10224            let agent_failure = class.is_some()
10225                && !nudge_first
10226                && !roster.is_empty()
10227                && next_agent(i, &tried[i], &specs).is_some();
10228            if failed && !quota && !agent_failure && nudges[i] < retries {
10229                nudges[i] += 1;
10230                still.push(i);
10231            }
10232        }
10233        pending = still;
10234    }
10235
10236    // One loss per seat whose chain ended on a quota: the intermediate ones
10237    // were absorbed by a handover and are not losses.
10238    for (i, q) in last_quota.into_iter().enumerate() {
10239        if let Some(reset) = q {
10240            losses.push(QuotaLoss {
10241                seat: originals[i].seat.key.clone(),
10242                node: ctx.node.to_owned(),
10243                at: Timestamp::now(),
10244                reset,
10245            });
10246        }
10247    }
10248
10249    seats
10250        .into_iter()
10251        .zip(done)
10252        .zip(nudges)
10253        .map(|((seat, res), attempts)| {
10254            (
10255                seat,
10256                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10257                attempts,
10258            )
10259        })
10260        .collect()
10261}
10262
10263/// Did the wave end without asking any agent (see [`block_wave`])?
10264fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10265    results
10266        .iter()
10267        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10268}
10269
10270/// Settle a wave that never spawned anything: every job answers
10271/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10272/// else about the seat (session, history, quota) is touched.
10273fn block_wave(
10274    jobs: Vec<SeatJob>,
10275    state: &mut RunState,
10276    node: &str,
10277    attempt: usize,
10278    why: &str,
10279) -> Vec<(usize, SeatState, AgentOutcome)> {
10280    state.resource_wait = None;
10281    state
10282        .active
10283        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10284    if let Err(e) = state.save() {
10285        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10286    }
10287    jobs.into_iter()
10288        .enumerate()
10289        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10290        .collect()
10291}
10292
10293/// Acquire the shared build cache's lease, waiting out contention within
10294/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10295/// unbounded wait is never acceptable).
10296///
10297/// A first, non-blocking check happens before ever waiting; if it finds the
10298/// lease busy, that fact is logged as a `verify` event *and* flushed with
10299/// [`RunState::save`] immediately — not only once the wait finally succeeds
10300/// or gives up — so a `magi show` run by a different process while this one
10301/// is still waiting reads a `run.json` that says so, rather than whatever it
10302/// looked like before the wait started. The same applies to the terminal
10303/// failure: logged and saved before this returns `Err`, so a caller that
10304/// could not get the lease at all still leaves a legible record of why.
10305async fn acquire_cache_lease(
10306    state: &mut RunState,
10307    cache_dir: &Path,
10308    owner: &crate::cache::Owner,
10309    budget: Duration,
10310    node: &str,
10311    context: &str,
10312    cancel: &(dyn Fn() -> bool + Sync),
10313) -> Result<Option<crate::cache::Guard>> {
10314    let home = crate::run::home();
10315    let started = Instant::now();
10316    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10317        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10318        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10319        Err(e) => {
10320            state.event(
10321                "verify",
10322                format!("{context}: could not check the shared build cache: {e:#}"),
10323            );
10324            if let Err(e2) = state.save() {
10325                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10326            }
10327            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10328        }
10329    };
10330    state.event(
10331        "verify",
10332        format!(
10333            "{context}: waiting for the shared build cache at {} ({})",
10334            cache_dir.display(),
10335            busy.describe()
10336        ),
10337    );
10338    // Seats of this node are waiting, not running: say so, so the countdown a
10339    // reader sees only ever measures an agent that has been spawned.
10340    state.resource_wait = Some(crate::run::ResourceWait {
10341        node: context.to_owned(),
10342        kind: "shared build cache".to_owned(),
10343        holder: busy.describe(),
10344        since: Timestamp::now(),
10345        limit_secs: budget.as_secs(),
10346    });
10347    if let Err(e) = state.save() {
10348        tracing::warn!("could not persist a cache wait: {e:#}");
10349    }
10350    let remaining = budget.saturating_sub(started.elapsed());
10351    let waited = crate::cache::wait_for_unless(
10352        &home,
10353        cache_dir,
10354        owner,
10355        remaining,
10356        Duration::from_secs(5),
10357        cancel,
10358    )
10359    .await;
10360    state.resource_wait = None;
10361    let cause = match &waited {
10362        Ok(Some(_)) => None,
10363        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10364        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10365            Some(crate::run::ResourceBlockCause::Timeout {
10366                waited_secs: started.elapsed().as_secs(),
10367                limit_secs: budget.as_secs(),
10368            })
10369        }
10370        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10371    };
10372    if let Some(cause) = cause {
10373        let detail = match &waited {
10374            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10375            Err(error) => format!("{error:#}"),
10376        };
10377        state.resource_block = Some(crate::run::ResourceBlock {
10378            cause,
10379            node: node.to_owned(),
10380            kind: "shared build cache".into(),
10381            detail,
10382            at: Timestamp::now(),
10383        });
10384        state.event(
10385            "resource",
10386            state.resource_block.as_ref().unwrap().description(),
10387        );
10388    }
10389    if let Err(error) = state.save() {
10390        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10391    }
10392    waited
10393}
10394
10395/// The lease state could not be read at all (unreadable lease file, a
10396/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10397/// known to be contended, so a wave degrades to running without a shared cache
10398/// instead of stopping on a hold that [`crate::triage`] could never release.
10399#[derive(Debug)]
10400struct CacheCheckError(String);
10401
10402impl std::fmt::Display for CacheCheckError {
10403    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10404        f.write_str(&self.0)
10405    }
10406}
10407
10408impl std::error::Error for CacheCheckError {}
10409
10410/// Keep a specific acquisition outcome when the caller adds command evidence.
10411fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10412    state
10413        .resource_block
10414        .get_or_insert_with(|| crate::run::ResourceBlock {
10415            cause: crate::run::ResourceBlockCause::Other,
10416            node: node.to_owned(),
10417            kind: "shared build cache".into(),
10418            detail: format!("{error:#}"),
10419            at: Timestamp::now(),
10420        });
10421    if let Err(error) = state.save() {
10422        tracing::warn!("could not persist cache resource block: {error:#}");
10423    }
10424}
10425
10426/// Run `body` — a verify command batch — while holding the shared build
10427/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10428/// never interleave with another borrower's build against the same
10429/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10430/// timeout, or a human's own `magi review`. See the `cache` module doc for
10431/// why this matters more than Cargo's own per-target locking covers — two
10432/// *different* worktrees building the same package name/version into one
10433/// cache directory is a staleness bug, not a lock contention one.
10434///
10435/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10436/// agent waves do. Repository scheduling also has its own bounded,
10437/// cancellable resource-wait policy. Contention must not consume the
10438/// verification budget: `body` receives the full `budget` after both leases
10439/// are acquired and the cache is fresh. Without a shared cache, only
10440/// repository coordination is acquired before handing over the full budget.
10441///
10442/// A lease that cannot be acquired within the resource wait is reported as a single
10443/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10444/// verification — the same shape a spawn failure already takes in
10445/// [`run_commands`], so a caller need not special-case it.
10446#[allow(clippy::too_many_arguments)]
10447async fn with_cache_lease<'s, F, Fut>(
10448    state: &'s mut RunState,
10449    cache_dir: Option<&Path>,
10450    node: &str,
10451    seat: &str,
10452    worktree: &Path,
10453    head: &str,
10454    budget: Duration,
10455    context: &str,
10456    wait: &crate::repo_lock::WaitPolicy,
10457    body: F,
10458) -> Result<(Vec<CommandOutcome>, bool)>
10459where
10460    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10461    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10462{
10463    // Match editing waves: cache first, repository second. In particular,
10464    // pre_gate must never hold the repository while waiting for a wave's cache.
10465    let started = Instant::now();
10466    let home = crate::run::home();
10467    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10468    let guard = if let Some(cache_dir) = cache_dir {
10469        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10470        match acquire_cache_lease(
10471            state,
10472            cache_dir,
10473            &owner,
10474            resource_limit,
10475            node,
10476            context,
10477            &|| wait.cancelled(),
10478        )
10479        .await
10480        .and_then(|g| {
10481            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10482        }) {
10483            Ok(g) => Some(g),
10484            Err(e) => {
10485                record_cache_error(state, node, &e);
10486                return Ok((
10487                    vec![CommandOutcome {
10488                        command: "(waiting for the shared build cache)".into(),
10489                        code: None,
10490                        output_tail: e.to_string(),
10491                        duration_ms: started.elapsed().as_millis() as u64,
10492                        resource_blocked: true,
10493                    }],
10494                    false,
10495                ));
10496            }
10497        }
10498    } else {
10499        None
10500    };
10501    let _coordination = coordinate_operation(state, worktree, wait).await?;
10502    if let Some(cache_dir) = cache_dir {
10503        let identity = crate::cache::Identity::new(worktree, head);
10504        if let Err(e) = crate::cache::ensure_fresh(
10505            &home,
10506            cache_dir,
10507            &identity,
10508            &state.config.verify.cargo_manifests(),
10509        ) {
10510            record_cache_error(state, node, &e);
10511            state.event(
10512                "verify",
10513                format!(
10514                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10515                    worktree.display(),
10516                    short(head)
10517                ),
10518            );
10519            if let Some(guard) = guard {
10520                guard.release();
10521            }
10522            return Ok((
10523                vec![CommandOutcome {
10524                    command: "(confirming the shared build cache is fresh)".into(),
10525                    code: None,
10526                    output_tail: e.to_string(),
10527                    duration_ms: started.elapsed().as_millis() as u64,
10528                    resource_blocked: true,
10529                }],
10530                false,
10531            ));
10532        }
10533    }
10534    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10535    if !timed_out_pids.is_empty() {
10536        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10537    }
10538    // Drop repository coordination before releasing the cache, reversing the
10539    // acquisition order. A cancelled/error future also drops both guards.
10540    drop(_coordination);
10541    if let Some(guard) = guard {
10542        guard.release();
10543    }
10544    Ok((outcomes, retried))
10545}
10546
10547/// Poll `pids` — commands [`run_commands`] reports as still running when its
10548/// own timeout elapsed — until every one is confirmed gone, or
10549/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10550///
10551/// Real confirmation where confirmation is possible, not a substitute for
10552/// full process-tree observation: a grandchild the timed-out process spawned
10553/// and that survives independently of it is invisible to a pid check the
10554/// same way it always was, and continuing to observe and collect *that*
10555/// stays a different piece of work with its own owner. This only narrows a
10556/// fixed blind wait into an actual check of the pids this process does know
10557/// about.
10558async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10559    wait_for_pids_with(
10560        pids,
10561        crate::proc::pid_alive,
10562        LEASE_RELEASE_POLL,
10563        LEASE_RELEASE_MAX_WAIT,
10564    )
10565    .await;
10566}
10567
10568/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10569/// interval and ceiling supplied by the caller, so the polling *logic* -
10570/// returns as soon as every pid reports dead, gives up at the ceiling
10571/// otherwise - is testable on millisecond durations without asking the real
10572/// OS about a pid at all.
10573async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10574    pids: &[u32],
10575    alive: F,
10576    poll: Duration,
10577    max_wait: Duration,
10578) {
10579    let deadline = Instant::now() + max_wait;
10580    loop {
10581        if pids.iter().all(|&pid| !alive(pid)) {
10582            return;
10583        }
10584        if Instant::now() >= deadline {
10585            return;
10586        }
10587        tokio::time::sleep(poll).await;
10588    }
10589}
10590
10591/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10592/// admission that it could not even get a verify command to run, as opposed
10593/// to evidence the command actually produced? A caller that would otherwise
10594/// read a resource-blocked outcome as a red command must check this first:
10595/// see [`Runner::gate`], which retries rather than records `Blocked` when
10596/// this is true.
10597fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10598    outcomes.iter().any(|o| o.resource_blocked)
10599}
10600
10601/// What [`Runner::gate_fix_round`] decided.
10602pub(crate) enum GateFix {
10603    /// The tree changed and `verify.e2e` is still green: run the gate again.
10604    Retry,
10605    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10606    /// last failure stands and the run ends blocked.
10607    Stop,
10608    /// `verify.e2e` could not run after the fix (magi's own contention):
10609    /// decide nothing now, a later reentry retries.
10610    Defer,
10611}
10612
10613/// Is every red command in `outcomes` an ordinary failure the code could
10614/// explain: it ran, exited non-zero, and said something?
10615///
10616/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10617/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10618/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10619/// what the command is stays the gate's business.
10620fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10621    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10622    red.peek().is_some()
10623        && red.all(|o| {
10624            !o.resource_blocked
10625                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10626                && !o.output_tail.trim().is_empty()
10627        })
10628}
10629
10630/// Describe one verify command's outcome for the event log, distinguishing a
10631/// build/link failure — the toolchain never produced a binary to run — from
10632/// an actual test failure, since only the latter is a verdict on the patch.
10633fn e2e_outcome_label(o: &CommandOutcome) -> String {
10634    if o.ok() {
10635        return "pass".to_owned();
10636    }
10637    let reason = if o.build_failed() {
10638        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10639    } else {
10640        format!("FAIL ({:?})", o.code)
10641    };
10642    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10643}
10644
10645/// Run `verify.e2e`, retrying once if the first attempt could not build or
10646/// link — a build/link failure is frequently a race against a shared
10647/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10648/// `verify` event per command, tagged with `context` (normally `"round N"`)
10649/// so the two call sites that need this — the ordinary per-round leg in
10650/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10651/// it will ever call a round green — read identically in the event log.
10652async fn run_e2e_with_retry(
10653    state: &mut RunState,
10654    shell: &[String],
10655    commands: &[String],
10656    worktree: &Path,
10657    timeout: Duration,
10658    context: &str,
10659) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10660    let (mut e2e, mut timed_out_pids) = run_commands(
10661        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10662    )
10663    .await;
10664    for o in &e2e {
10665        state.event(
10666            "verify",
10667            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10668        );
10669    }
10670    // A build/link failure is not a verdict on the patch — it is frequently a
10671    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10672    // one retry before letting a red like that decide the round.
10673    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10674    if verify_retried {
10675        state.event(
10676            "verify",
10677            format!(
10678                "{context}: verify could not build/link, not a test result — retrying once \
10679                 before concluding"
10680            ),
10681        );
10682        let retried = run_commands(
10683            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10684        )
10685        .await;
10686        e2e = retried.0;
10687        // Both attempts' timeouts matter, not just the last one: the first
10688        // attempt's descendants may still be alive alongside the retry's.
10689        timed_out_pids.extend(retried.1);
10690        for o in &e2e {
10691            state.event(
10692                "verify",
10693                format!(
10694                    "{context}: retry `{}` -> {}",
10695                    o.command,
10696                    e2e_outcome_label(o)
10697                ),
10698            );
10699        }
10700    }
10701    (e2e, verify_retried, timed_out_pids)
10702}
10703
10704/// Run configured shell commands in `cwd`, in order. The second element is
10705/// the pid of every command that hit `timeout` and was still running when
10706/// this stopped waiting on it (best-effort: `None` when the platform did not
10707/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10708/// that releases a shared resource afterward needs to know.
10709///
10710/// Records `task` into [`RunState::active`] at every command boundary
10711/// (`RunState::task_command`) and clears it once the whole list has run
10712/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10713/// for minutes with no seat and no output of its own to show for it (see
10714/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10715/// apart from "not yet run" without this), and this is the only place that
10716/// knows which command is running right now and how many are left. Three
10717/// saves per command — start, not per second — matching the same "only at a
10718/// boundary" rule [`wave`] already follows for seats.
10719#[allow(clippy::too_many_arguments)]
10720async fn run_commands(
10721    state: &mut RunState,
10722    node: &str,
10723    task: &str,
10724    attempt: usize,
10725    shell: &[String],
10726    commands: &[String],
10727    cwd: &Path,
10728    timeout: Duration,
10729) -> (Vec<CommandOutcome>, Vec<u32>) {
10730    if commands.is_empty() {
10731        // Nothing to mark as running and nothing to clear — an empty list
10732        // means "not configured", and touching `active` (or the disk) over
10733        // that would be a write for every round of a repo with no
10734        // `verify.e2e` / `verify.gate` commands at all.
10735        return (Vec::new(), Vec::new());
10736    }
10737    let mut out = Vec::new();
10738    let mut timed_out_pids = Vec::new();
10739    let total = commands.len();
10740    for (idx, command) in commands.iter().enumerate() {
10741        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10742        if let Err(e) = state.save() {
10743            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10744        }
10745        let started = Instant::now();
10746        let mut cmd = tokio::process::Command::new(&shell[0]);
10747        cmd.quiet();
10748        // Verify the checkout as a standalone caller. Worker ownership and
10749        // pinned executable selection belong to this driver, not to tests or
10750        // tools it runs. Preserve home/seat context and Running's containment.
10751        for key in crate::worker::INVOCATION_ENV {
10752            cmd.env_remove(key);
10753        }
10754        cmd.args(&shell[1..])
10755            .arg(command)
10756            .current_dir(cwd)
10757            .stdin(std::process::Stdio::null())
10758            .stdout(std::process::Stdio::piped())
10759            .stderr(std::process::Stdio::piped())
10760            .kill_on_drop(true);
10761        let mut outputs = Vec::new();
10762        let protected = (|| -> Result<()> {
10763            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10764                outputs.push(guard);
10765            }
10766            if let Some(cache) = state.config.cache_dir() {
10767                outputs.push(crate::retention::OutputLease::cache(
10768                    &crate::run::home(),
10769                    &cache,
10770                    false,
10771                )?);
10772            }
10773            Ok(())
10774        })();
10775        if let Err(e) = protected {
10776            out.push(CommandOutcome {
10777                command: command.clone(),
10778                code: None,
10779                output_tail: format!("output ownership unavailable: {e:#}"),
10780                duration_ms: 0,
10781                resource_blocked: true,
10782            });
10783            continue;
10784        }
10785        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10786        let (code, body) = match spawned {
10787            Ok(mut running) => {
10788                let pid = running.child().id();
10789                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10790                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10791                let mut paths = vec![cwd.to_owned()];
10792                if let Some(cache) = state.config.cache_dir() {
10793                    paths.push(cache);
10794                }
10795                let mut pressure = None;
10796                let status = tokio::select! {
10797                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10798                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10799                        pressure = Some(reason); None
10800                    }
10801                };
10802                let (code, failure) = match status {
10803                    Some(Ok(Ok(status))) => (
10804                        status.code(),
10805                        status
10806                            .code()
10807                            .is_none()
10808                            .then(|| format!("command terminated: {status}")),
10809                    ),
10810                    Some(Ok(Err(error))) => {
10811                        (None, Some(format!("failed waiting for command: {error}")))
10812                    }
10813                    Some(Err(_)) => {
10814                        // Leased cleanup confirms process-tree death itself.
10815                        // Unleased commands still need the caller's PID check
10816                        // after kill-on-drop, before releasing cache ownership.
10817                        if !running.coordinated()
10818                            && let Some(pid) = pid
10819                        {
10820                            timed_out_pids.push(pid);
10821                        }
10822                        (
10823                            None,
10824                            Some(format!("timed out after {}s", timeout.as_secs())),
10825                        )
10826                    }
10827                    None => (None, None),
10828                };
10829                running.finish().await;
10830                if let Some(reason) = pressure {
10831                    state.resource_block = Some(crate::run::ResourceBlock {
10832                        cause: crate::run::ResourceBlockCause::Other,
10833                        node: node.to_owned(),
10834                        kind: "disk pressure".to_owned(),
10835                        detail: reason.clone(),
10836                        at: Timestamp::now(),
10837                    });
10838                    out.push(CommandOutcome {
10839                        command: command.clone(),
10840                        code: None,
10841                        output_tail: reason,
10842                        duration_ms: started.elapsed().as_millis() as u64,
10843                        resource_blocked: true,
10844                    });
10845                    break;
10846                }
10847                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10848                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10849                if let Some(failure) = failure {
10850                    body.push('\n');
10851                    body.push_str(&failure);
10852                }
10853                (code, body)
10854            }
10855            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10856        };
10857        out.push(CommandOutcome {
10858            command: command.clone(),
10859            code,
10860            output_tail: tail(&body, OUTPUT_TAIL),
10861            duration_ms: started.elapsed().as_millis() as u64,
10862            resource_blocked: false,
10863        });
10864    }
10865    state.task_finished(task);
10866    if let Err(e) = state.save() {
10867        tracing::warn!("could not persist the end of {task}: {e:#}");
10868    }
10869    (out, timed_out_pids)
10870}
10871
10872/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10873/// section (`report::run`) and in the `merge` event this node records — for
10874/// the operator to run by hand.
10875///
10876/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10877/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10878/// merge commits", or "require linear history") rejects the push a `--no-ff`
10879/// merge would produce, which is exactly the guidance this function replaces.
10880/// `message`'s first line becomes the squash commit's subject, matching the
10881/// note `report::run` prints alongside this command — see that function for
10882/// why an explicit subject is not optional there.
10883fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10884    let repo = repo.display();
10885    match style {
10886        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10887        MergeStyle::Squash => {
10888            // The subject sits inside double quotes, and a title an agent
10889            // wrote may carry the characters that break out of them.
10890            let subject = message
10891                .lines()
10892                .next()
10893                .unwrap_or(branch)
10894                .replace(['\\', '"', '$', '`'], "");
10895            format!(
10896                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10897            )
10898        }
10899        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10900    }
10901}
10902
10903/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10904/// under the hood, rejects a `title` over 256 characters and the whole
10905/// command fails — no PR at all, for a run whose body was otherwise fine
10906/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10907/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10908/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10909/// this bound. It is a margin, not a guarantee — a title packed
10910/// with multi-unit characters could still in principle land close to the
10911/// edge, but a real task title's occasional emoji or accented letter fits
10912/// comfortably inside it.
10913const PR_TITLE_MAX: usize = 240;
10914
10915/// A pull request title from the opening line of `text`, or `None` when that
10916/// line is not English (GitHub text is) or has no letters.
10917///
10918/// A line within `max` is kept as is. A longer one is cut at the end of its
10919/// first sentence when that falls inside `max`, else at a word boundary with a
10920/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10921/// make every merely-truncated title look non-English). The language check
10922/// runs on the kept text before any mark is added, so only what GitHub will
10923/// show is judged: an English opening followed by non-ASCII far past the cut
10924/// still passes.
10925fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10926    let line = queue::first_line(text)?;
10927    let chars: Vec<char> = line.chars().collect();
10928    let (kept, mark) = if chars.len() <= max {
10929        (line.to_owned(), "")
10930    } else if let Some(end) = sentence_end(&chars, max) {
10931        (chars[..end].iter().collect::<String>(), "")
10932    } else {
10933        let room = max.saturating_sub(3);
10934        // Cut at the last space inside the room; when the char just past the
10935        // room is a space the room already ends on a word.
10936        let cut = if chars[room].is_whitespace() {
10937            room
10938        } else {
10939            chars[..room]
10940                .iter()
10941                .rposition(|c| c.is_whitespace())
10942                .unwrap_or(room)
10943        };
10944        let head: String = chars[..cut].iter().collect();
10945        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10946        (head.to_owned(), "...")
10947    };
10948    if kept.is_empty() {
10949        return None;
10950    }
10951    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10952        return None;
10953    }
10954    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10955        return None;
10956    }
10957    Some(format!("{kept}{mark}"))
10958}
10959
10960/// The char length of the first sentence of `chars` when it ends within `max`
10961/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10962/// abbreviations so `e.g. foo` does not end a title early.
10963fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10964    const MIN: usize = 20;
10965    for i in MIN..max.min(chars.len()) {
10966        if !matches!(chars[i], '.' | '!' | '?') {
10967            continue;
10968        }
10969        let Some(&next) = chars.get(i + 1) else {
10970            continue;
10971        };
10972        if !next.is_whitespace() {
10973            continue;
10974        }
10975        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10976        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10977            continue;
10978        }
10979        let word: String = chars[..i]
10980            .iter()
10981            .rev()
10982            .take_while(|c| !c.is_whitespace())
10983            .collect::<Vec<_>>()
10984            .into_iter()
10985            .rev()
10986            .collect();
10987        let word = word.to_ascii_lowercase();
10988        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10989            continue;
10990        }
10991        let end = chars[..i]
10992            .iter()
10993            .rposition(|c| !c.is_whitespace())
10994            .map_or(i, |p| p + 1);
10995        return Some(end);
10996    }
10997    None
10998}
10999
11000/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11001/// change: a title and a body describing what was *implemented*, not the task
11002/// that asked for it. A task reads as a request; a reader of the merged
11003/// history wants the change.
11004struct PrMessage {
11005    title: String,
11006    body: String,
11007}
11008
11009impl PrMessage {
11010    /// Title, blank line, body. The first line is the squash/merge commit
11011    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11012    /// has to stay one sensible line.
11013    fn commit_message(&self) -> String {
11014        format!("{}\n\n{}", self.title, self.body)
11015    }
11016}
11017
11018/// The text after a leading `TITLE:` (any case) on `line`.
11019fn title_marker(line: &str) -> Option<&str> {
11020    let line = line.trim();
11021    let head = line.get(..6)?;
11022    head.eq_ignore_ascii_case("title:")
11023        .then(|| line[6..].trim())
11024}
11025
11026/// The implementer's own one-line title: the `TITLE:` line the implement
11027/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11028/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11029/// source, and a title that says as much is refused here too.
11030fn summary_title(summary: &str) -> Option<String> {
11031    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11032    let raw = title_marker(first)?;
11033    if raw.is_empty() {
11034        return None;
11035    }
11036    let title = queue::title_from(raw, PR_TITLE_MAX);
11037    let lower = title.to_ascii_lowercase();
11038    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11039        return None;
11040    }
11041    Some(title)
11042}
11043
11044/// How `open_review`'s instruction begins; see [`landing_title`].
11045const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11046
11047/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11048pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11049
11050/// Does this run review work that already existed, rather than implement a
11051/// task? Runs recorded before `reviewed_commits` existed carry only the
11052/// candidate marker.
11053pub(crate) fn is_review_run(state: &RunState) -> bool {
11054    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11055}
11056
11057/// The title of a review-only run: the subject of the oldest commit under
11058/// review. Later commits are usually fixups, and `instruction` is the review
11059/// prompt, which says nothing about the change. GitHub text is English, so a
11060/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11061fn review_title(state: &RunState) -> Option<String> {
11062    english_subject(
11063        state.reviewed_commits.as_ref()?.first()?,
11064        state.config.graph.github_english(),
11065    )
11066}
11067
11068/// `raw` as a pull request title, or `None` when it is blank, not English
11069/// (GitHub text is), or one of magi's own candidate commit subjects.
11070fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11071    let raw = raw.trim();
11072    if raw.is_empty() {
11073        return None;
11074    }
11075    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11076        return None;
11077    }
11078    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11079        return None;
11080    }
11081    let title = queue::title_from(raw, PR_TITLE_MAX);
11082    let lower = title.to_ascii_lowercase();
11083    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11084        return None;
11085    }
11086    Some(title)
11087}
11088
11089/// What a review-only run's branch says about itself, read at the moment the
11090/// pull request is opened.
11091#[derive(Debug, Clone, PartialEq, Eq)]
11092struct BranchFacts {
11093    /// `(subject, body)` of each commit, oldest first.
11094    commits: Vec<(String, String)>,
11095    /// Trimmed `git diff --stat`.
11096    stat: String,
11097}
11098
11099/// Longest diff stat shown: this many file lines plus the summary line.
11100const STAT_FILE_LINES: usize = 25;
11101/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11102const COMMITS_MAX_CHARS: usize = 20_000;
11103
11104/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11105/// say or finds nothing, so the caller falls back to what the run recorded.
11106async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11107    let commits = git::commit_log(repo, base, branch).await.ok()?;
11108    if commits.is_empty() {
11109        return None;
11110    }
11111    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11112    let lines: Vec<&str> = stat.lines().collect();
11113    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11114        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11115        let more = format!(" ... {omitted} more file(s)");
11116        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11117        kept.push(&more);
11118        kept.push(lines[lines.len() - 1]);
11119        kept.join("\n")
11120    } else {
11121        lines.join("\n")
11122    };
11123    Some(BranchFacts { commits, stat })
11124}
11125
11126/// Defang what would break the surrounding markdown: a closing `</details>`
11127/// and a code fence.
11128fn markdown_safe(text: &str) -> String {
11129    text.replace("</details>", "&lt;/details&gt;")
11130        .replace("\x60\x60\x60", "~~~")
11131}
11132
11133fn neutral_title(state: &RunState, winner: char) -> String {
11134    format!(
11135        "chore: land candidate {} of run {}",
11136        winner.to_ascii_uppercase(),
11137        state.id
11138    )
11139}
11140
11141/// Reuse the graph's bounded fixer without starting another graph or driver.
11142/// Copy the working state back even on errors so spent rounds remain visible.
11143pub(crate) async fn repair_landing_gate(
11144    state: &mut RunState,
11145    winner: &Candidate,
11146    outcomes: &[CommandOutcome],
11147) -> Result<GateFix> {
11148    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11149        return Ok(GateFix::Stop);
11150    }
11151    let mut runner = Runner {
11152        roles: state.config.resolve_roles()?,
11153        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11154        state: state.clone(),
11155        pause: Pause::new(),
11156        interrupt: Pause::new(),
11157    };
11158    let result = runner.gate_fix_round(winner, outcomes).await;
11159    *state = runner.state;
11160    result
11161}
11162
11163/// Recheck a repair's e2e requirement across failures, resource deferrals and
11164/// process interruptions. A successful result only applies to its exact SHA.
11165pub(crate) async fn check_gate_repair_e2e(
11166    state: &mut RunState,
11167    winner: &Candidate,
11168    wait: &crate::repo_lock::WaitPolicy,
11169) -> Result<GateFix> {
11170    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11171        return Ok(GateFix::Retry);
11172    }
11173    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11174    if state
11175        .gate_repair_e2e
11176        .as_ref()
11177        .is_some_and(|r| r.head == head && r.passed)
11178    {
11179        return Ok(GateFix::Retry);
11180    }
11181    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11182        head: head.clone(),
11183        passed: false,
11184        outcomes: Vec::new(),
11185    });
11186    state.save()?;
11187    let commands = state.config.verify.e2e.clone();
11188    let outcomes = if commands.is_empty() {
11189        Vec::new()
11190    } else {
11191        let shell = state.config.shell();
11192        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11193        let cache_dir = state.config.cache_dir();
11194        let (outcomes, _) = with_cache_lease(
11195            state,
11196            cache_dir.as_deref(),
11197            "e2e",
11198            "e2e",
11199            &winner.worktree,
11200            &head,
11201            timeout,
11202            "gate repair e2e",
11203            wait,
11204            |state, budget| async move {
11205                run_e2e_with_retry(
11206                    state,
11207                    &shell,
11208                    &commands,
11209                    &winner.worktree,
11210                    budget,
11211                    "gate repair e2e",
11212                )
11213                .await
11214            },
11215        )
11216        .await?;
11217        outcomes
11218    };
11219    let deferred = verify_inconclusive(&outcomes);
11220    let passed = outcomes.iter().all(CommandOutcome::ok)
11221        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11222    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11223        head,
11224        passed,
11225        outcomes,
11226    });
11227    state.save()?;
11228    if deferred {
11229        Ok(GateFix::Defer)
11230    } else if !passed {
11231        state.event(
11232            "gate",
11233            "verify.e2e failed after gate repair; nothing published",
11234        );
11235        Ok(GateFix::Stop)
11236    } else {
11237        Ok(GateFix::Retry)
11238    }
11239}
11240
11241/// Run the same configured gate for a landing head and a fresh graph.
11242pub(crate) async fn run_landing_gate(
11243    state: &mut RunState,
11244    winner: &Candidate,
11245    wait: &crate::repo_lock::WaitPolicy,
11246) -> Result<Vec<CommandOutcome>> {
11247    let shell = state.config.shell();
11248    let gate_commands = state.config.verify.gate.clone();
11249    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11250    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11251        Vec::new()
11252    } else {
11253        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11254        let cache_dir = state.config.cache_dir();
11255        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11256        let (outcomes, _) = with_cache_lease(
11257            state,
11258            cache_dir.as_deref(),
11259            "gate",
11260            "gate",
11261            &winner.worktree,
11262            &head,
11263            timeout,
11264            "final gate",
11265            wait,
11266            |state, budget| {
11267                let shell = shell.clone();
11268                let gate_commands = gate_commands.clone();
11269                let winner = winner.clone();
11270                async move {
11271                    Runner::run_pre_gate(state, &winner).await;
11272                    let (outcomes, timed_out_pids) = run_commands(
11273                        state,
11274                        "gate",
11275                        "gate",
11276                        0,
11277                        &shell,
11278                        &gate_commands,
11279                        &winner.worktree,
11280                        budget,
11281                    )
11282                    .await;
11283                    (outcomes, false, timed_out_pids)
11284                }
11285            },
11286        )
11287        .await?;
11288        outcomes
11289    };
11290    if outcomes.is_empty() {
11291        // Nothing configured to check — distinct from every other
11292        // silence in this run's event log, since an empty `gate` alone
11293        // no longer says whether the gate ran at all (see
11294        // `RunState::gate_ran`'s own doc).
11295        state.event("gate", "no gate commands configured; not tested");
11296    }
11297    for o in &outcomes {
11298        state.event(
11299            "gate",
11300            format!(
11301                "`{}` -> {}",
11302                o.command,
11303                if o.ok() {
11304                    "pass".to_owned()
11305                } else {
11306                    format!(
11307                        "FAIL ({:?})\n{}",
11308                        o.code,
11309                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11310                    )
11311                }
11312            ),
11313        );
11314    }
11315    Ok(outcomes)
11316}
11317
11318/// The pull request title to hand to `land::merge_subject`. A review-only run
11319/// opened by an earlier build titled its pull request with the review prompt;
11320/// that title is dropped (empty, so the fallback applies) rather than landed.
11321/// Any other title, including an operator's rename, passes through untouched,
11322/// and so does every title of a run that implements a task.
11323pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11324    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11325        ""
11326    } else {
11327        pr_title
11328    }
11329}
11330
11331/// What the squash subject falls back to when the pull request title is empty
11332/// or candidate-shaped: for a review-only run the derived title, never the
11333/// review prompt held in `instruction`.
11334pub fn landing_subject_source(state: &RunState) -> String {
11335    if is_review_run(state) {
11336        let winner = state.candidates.first().map_or('A', |c| c.label);
11337        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11338    }
11339    state.instruction.clone()
11340}
11341
11342/// `summary` without its `TITLE:` line, which the pull request title already
11343/// carries.
11344fn summary_without_title(summary: &str) -> String {
11345    let mut lines = summary.trim().lines().peekable();
11346    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11347        lines.next();
11348    }
11349    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11350}
11351
11352/// The pull request title and body for the winning candidate.
11353///
11354/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11355/// the task's own opening line via [`queue::title_from`] when there is none.
11356/// `state.instruction` can open with blank lines (`task_text` only rejects a
11357/// body that is blank *entirely*), which `title_from` skips.
11358///
11359/// Publishing uses an independent description generated from the final diff
11360/// and structured evidence. Internal summaries and reports remain local.
11361/// Non-publishing previews may display the original task and candidate summary.
11362/// The footer retains the run and candidate provenance.
11363#[cfg(test)]
11364fn pr_message(state: &RunState, winner: char) -> PrMessage {
11365    pr_message_with(state, winner, None)
11366}
11367
11368/// [`pr_message`] with what the branch of a review-only run says about itself.
11369/// `facts` is ignored for a run that implements a task.
11370#[cfg(test)]
11371fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11372    let raw = pr_message_raw(
11373        state,
11374        winner,
11375        facts,
11376        crate::pr_outcome::FinalHead::Unreported,
11377        ChangeNote::NotAsked,
11378    );
11379    let id = crate::scrub::Identity::current();
11380    PrMessage {
11381        title: crate::scrub::scrub(&raw.title, &id),
11382        body: crate::scrub::scrub(&raw.body, &id),
11383    }
11384}
11385
11386/// What the public description may say about the change itself.
11387#[derive(Debug, Clone, Copy)]
11388enum ChangeNote<'a> {
11389    /// A local preview or non-publishing merge mode: no public generation needed.
11390    NotAsked,
11391    /// A summary written from the final diff.
11392    Summary(&'a crate::run::ChangeSummary),
11393    /// A summary was asked for and could not be produced; the text says why.
11394    Unavailable(&'a str),
11395}
11396
11397fn pr_message_raw(
11398    state: &RunState,
11399    winner: char,
11400    facts: Option<&BranchFacts>,
11401    head: crate::pr_outcome::FinalHead<'_>,
11402    change: ChangeNote<'_>,
11403) -> PrMessage {
11404    if !matches!(change, ChangeNote::NotAsked) {
11405        let (title, mut body) = match change {
11406            ChangeNote::Summary(c) => (
11407                c.title.clone(),
11408                format!("## Summary\n\n{}\n\n", c.body.trim()),
11409            ),
11410            ChangeNote::Unavailable(why) => (
11411                neutral_title(state, winner),
11412                format!(
11413                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11414                ),
11415            ),
11416            ChangeNote::NotAsked => unreachable!(),
11417        };
11418        body.push_str(&public_pr_evidence(state, head));
11419        body.push_str(&format!(
11420            "\n---\nmagi:run/{} magi:candidate-{}\n",
11421            state.id,
11422            winner.to_ascii_lowercase()
11423        ));
11424        return PrMessage { title, body };
11425    }
11426    pr_preview(state, winner, facts, head)
11427}
11428
11429/// Append public facts from the run records after every prose generation,
11430/// including posting-gate rewrites. The agent cannot drop this provenance.
11431fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11432    let mut body = crate::pr_outcome::review_rounds(state);
11433    body.push_str(&crate::pr_outcome::final_verification(state, head));
11434    let open = state.last_round_findings();
11435    if !open.is_empty() {
11436        body.push_str("## Open review findings\n\n");
11437        for f in open {
11438            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11439        }
11440        body.push('\n');
11441    }
11442    if let Some(c) = state.change_summary.as_ref()
11443        && head == crate::pr_outcome::FinalHead::At(&c.head)
11444    {
11445        body.push_str(&format!(
11446            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11447            c.base, c.head
11448        ));
11449    }
11450    body
11451}
11452
11453/// Local-only preview of the operator's task and candidate summary.
11454fn pr_preview(
11455    state: &RunState,
11456    winner: char,
11457    facts: Option<&BranchFacts>,
11458    head: crate::pr_outcome::FinalHead<'_>,
11459) -> PrMessage {
11460    let summary = state
11461        .candidates
11462        .iter()
11463        .find(|c| c.label == winner)
11464        .map(|c| c.summary.as_str())
11465        .unwrap_or_default();
11466    // The fallback is the operator's own words and may not be English; GitHub
11467    // text always is, so a non-English task gets a neutral title instead.
11468    let review = is_review_run(state);
11469    let title = if review {
11470        facts
11471            .and_then(|f| {
11472                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11473            })
11474            .or_else(|| review_title(state))
11475            .or_else(|| {
11476                state
11477                    .candidates
11478                    .iter()
11479                    .find(|c| c.label == winner)
11480                    .filter(|c| !c.branch.starts_with("magi/"))
11481                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11482            })
11483            .unwrap_or_else(|| neutral_title(state, winner))
11484    } else {
11485        summary_title(summary).unwrap_or_else(|| {
11486            english_title(
11487                &state.instruction,
11488                PR_TITLE_MAX,
11489                state.config.graph.github_english(),
11490            )
11491            .unwrap_or_else(|| neutral_title(state, winner))
11492        })
11493    };
11494
11495    let mut body = String::new();
11496    let what = summary_without_title(summary);
11497    if !what.is_empty() {
11498        body.push_str("## Summary\n\n");
11499        body.push_str(&what);
11500        body.push_str("\n\n");
11501    }
11502
11503    // Review history uses structured facts across all rounds, never reports.
11504    body.push_str(&crate::pr_outcome::review_rounds(state));
11505    body.push_str(&crate::pr_outcome::final_verification(state, head));
11506    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11507
11508    let open = state.open_findings();
11509    if !open.is_empty() {
11510        body.push_str("## Open review findings\n\n");
11511        for f in &open {
11512            body.push_str(&format!(
11513                "- `{}` [{:?}] {}\n",
11514                f.id, f.severity, "Context retained in local review records."
11515            ));
11516        }
11517        body.push('\n');
11518    }
11519
11520    if let Some(fix) = fix
11521        && !fix.rejected.is_empty()
11522    {
11523        body.push_str("## Declined by the fixer\n\n");
11524        for r in &fix.rejected {
11525            body.push_str(&format!(
11526                "- `{}`: {}\n",
11527                r.id, "Reason retained in local review records."
11528            ));
11529        }
11530        body.push('\n');
11531    }
11532
11533    if review {
11534        // The review prompt is not the task; list what the branch carries.
11535        body.push_str("## Commits under review\n\n");
11536        if let Some(facts) = facts {
11537            let mut left = COMMITS_MAX_CHARS;
11538            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11539                let mut entry = format!("- {}\n", markdown_safe(subject));
11540                for l in markdown_safe(text).lines() {
11541                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11542                }
11543                if left == 0 {
11544                    body.push_str(&format!(
11545                        "- ... {} more commit(s)\n",
11546                        facts.commits.len() - i
11547                    ));
11548                    break;
11549                }
11550                if entry.len() > left {
11551                    // Even the first commit is cut: one huge body must not
11552                    // push the whole description past GitHub's limit.
11553                    let mut end = left;
11554                    while !entry.is_char_boundary(end) {
11555                        end -= 1;
11556                    }
11557                    entry.truncate(end);
11558                    entry.push_str("\n  ... (truncated)\n");
11559                    left = 0;
11560                } else {
11561                    left -= entry.len();
11562                }
11563                body.push_str(&entry);
11564            }
11565            if !facts.stat.trim().is_empty() {
11566                body.push_str(&format!(
11567                    "\n## Diff stat\n\n```\n{}\n```\n",
11568                    markdown_safe(facts.stat.trim())
11569                ));
11570            }
11571        } else {
11572            match &state.reviewed_commits {
11573                Some(subjects) => {
11574                    for s in subjects {
11575                        body.push_str(&format!("- {}\n", s.trim()));
11576                    }
11577                }
11578                None => {
11579                    // An older run kept only the prompt, with the commit list
11580                    // after its first paragraph.
11581                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11582                    body.push_str(rest.trim());
11583                    body.push('\n');
11584                }
11585            }
11586        }
11587    } else {
11588        let task = state.instruction.trim();
11589        let task = if task.is_empty() {
11590            "(empty task)"
11591        } else {
11592            task
11593        };
11594        body.push_str(&format!(
11595            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11596            task.replace("</details>", "&lt;/details&gt;")
11597        ));
11598    }
11599
11600    body.push_str(&format!(
11601        "\n---\nmagi:run/{} magi:candidate-{}\n",
11602        state.id,
11603        winner.to_ascii_lowercase()
11604    ));
11605
11606    PrMessage { title, body }
11607}
11608
11609/// Source imports have already been prepared from structured inputs. Legacy
11610/// inferred references must never instruct a seat to reconstruct those imports.
11611fn seeded_instruction(state: &RunState) -> String {
11612    let mut inputs = Vec::new();
11613    if let Some(start) = &state.sources.start_from {
11614        inputs.push(format!(
11615            "Start from `{}` at `{}` (including ancestry).",
11616            start.token, start.sha
11617        ));
11618    }
11619    for pick in &state.sources.pick_commits {
11620        inputs.push(format!(
11621            "Pick `{}` at `{}` (single commit).",
11622            pick.token, pick.sha
11623        ));
11624    }
11625    if inputs.is_empty() {
11626        state.instruction.clone()
11627    } else {
11628        format!(
11629            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11630            state.instruction,
11631            inputs.join("\n")
11632        )
11633    }
11634}
11635
11636/// Does the winner have no commits ahead of the base it would land on?
11637/// Any failure to find out reads as "not empty": the merge then behaves as it
11638/// always did rather than refusing on a guess.
11639async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11640    let base = &state.base_branch;
11641    let mut against = base.clone();
11642    if mode == MergeMode::Pr {
11643        // A pull request lands on the remote's base, never the local branch
11644        // of the same name: if that cannot be read, "not empty" is the safe
11645        // answer.
11646        let remote = &state.config.merge.remote;
11647        let tracking = format!("{remote}/{base}");
11648        let fetched = git::fetch(repo, remote, base).await;
11649        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11650            against = tracking;
11651        } else {
11652            return false;
11653        }
11654    }
11655    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11656}
11657
11658/// Why nothing was opened for an empty winner, with what the task's own
11659/// references resolved to.
11660fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11661    let mut detail = format!(
11662        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11663         nothing to open a pull request for"
11664    );
11665    detail.push_str(
11666        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11667    );
11668    detail
11669}
11670
11671/// What the `Pr` merge does once it knows whether the branch already has an
11672/// open pull request.
11673#[derive(Debug, PartialEq, Eq)]
11674enum PrPlan {
11675    Create,
11676    Adopt { url: String, title: String },
11677    Stop(String),
11678}
11679
11680/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11681/// a failed lookup -> stop with the real reason. Never guesses.
11682fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11683    match found {
11684        Ok(land::OpenPr::None) => PrPlan::Create,
11685        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11686        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11687            "several open pull requests exist for this branch, not picking one: {}",
11688            urls.join(" ")
11689        )),
11690        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11691    }
11692}
11693
11694/// `gh pr create`, returning the PR url.
11695pub(crate) async fn gh_pr_create(
11696    cwd: &Path,
11697    base: &str,
11698    head: &str,
11699    title: &str,
11700    body: &str,
11701) -> Result<String> {
11702    let out = tokio::process::Command::new("gh")
11703        .args([
11704            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11705        ])
11706        .current_dir(cwd)
11707        .quiet()
11708        .stdin(std::process::Stdio::null())
11709        .output()
11710        .await
11711        .context("spawn gh")?;
11712    if out.status.success() {
11713        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11714    } else {
11715        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11716    }
11717}
11718
11719/// Never force deletion of uncommitted work, including unknown legacy trees.
11720async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11721    match std::fs::symlink_metadata(path) {
11722        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11723            // A previous cleanup may have removed the tree before a branch
11724            // deletion or state save failed. Unregister it if still present.
11725            git::worktree_remove_clean(repo, path).await?;
11726            return Ok(());
11727        }
11728        Err(e) => return Err(e.into()),
11729        Ok(_) => {}
11730    }
11731    if !git::is_clean(path).await? {
11732        bail!(
11733            "worktree has uncommitted work; refusing removal: {}",
11734            path.display()
11735        );
11736    }
11737    if !git::worktree_remove_clean(repo, path).await? {
11738        bail!("worktree removal refused: {}", path.display());
11739    }
11740    Ok(())
11741}
11742
11743/// Tear a run's worktrees and branches down.
11744///
11745/// `home` is where the updated `run.json` is saved (via
11746/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11747/// a housekeeping pass already has its own honest `home` handed to it, and
11748/// falling through to the global here would write back through whichever
11749/// directory some other process or test pinned into that `OnceLock` first,
11750/// not the one the caller actually resolved its `runs` and `state` from.
11751pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11752    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11753        bail!("refusing fold: {reason}");
11754    }
11755    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11756        bail!("candidate preservation incomplete; refusing cleanup");
11757    }
11758    // Serialize ordinary folding with the same starts/resumes as target-only
11759    // eviction. Otherwise a second janitor could remove the whole worktree
11760    // around a live output borrower even though target eviction was fenced.
11761    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11762    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11763    let _driver = crate::driver::maintenance(home, &state.id)?;
11764    let mut output_guards = Vec::new();
11765    for entry in std::fs::read_dir(state.worktree_root())
11766        .into_iter()
11767        .flatten()
11768        .flatten()
11769    {
11770        if entry.path().join(".git").is_file() {
11771            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11772        }
11773    }
11774    let repo = state.repo.clone();
11775    // CLI, Web and automatic cleanup all enter here, independently of
11776    // Runner::fold_losers. Protect deletion and shared config release too.
11777    let wait = repository_wait(state, None);
11778    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11779        Ok(lease) => lease,
11780        Err(e) => {
11781            state.save_under(home)?;
11782            return Err(e);
11783        }
11784    };
11785    // Confirmed merges dispose of their owned sources after durable evidence
11786    // capture. Preflight every source before removing anything.
11787    let _source_outputs = if state.status == RunStatus::Merged
11788        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11789    {
11790        verify_merged_fold_sources(state, drop_winner, home).await?
11791    } else {
11792        Vec::new()
11793    };
11794    let root = state.worktree_root();
11795    let winner = state.tally.as_ref().map(|t| t.winner);
11796    let mut removed = Vec::new();
11797
11798    for i in 0..state.candidates.len() {
11799        let c = state.candidates[i].clone();
11800        let is_winner = Some(c.label) == winner;
11801        if is_winner && !drop_winner {
11802            continue;
11803        }
11804        let existed = c.worktree.exists();
11805        // A vanished directory can still have a Git registration that keeps
11806        // its branch checked out. Remove that registration before the branch.
11807        remove_preserved_worktree(&repo, &c.worktree).await?;
11808        if existed {
11809            removed.push(c.worktree.to_string_lossy().into_owned());
11810        }
11811        // A branch handed to a later run (and its pull request) is not this
11812        // run's to delete.
11813        let handed_over = state.released_branches.contains(&c.branch);
11814        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11815            if !git::branch_delete(&repo, &c.branch).await? {
11816                bail!("candidate branch removal refused");
11817            }
11818            removed.push(c.branch.clone());
11819        }
11820        state.candidates[i].folded = true;
11821    }
11822
11823    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11824        let path = name.path();
11825        let keep = !drop_winner
11826            && winner.is_some_and(|w| {
11827                path.file_name()
11828                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11829            });
11830        if keep {
11831            continue;
11832        }
11833        remove_preserved_worktree(&repo, &path).await?;
11834        removed.push(path.to_string_lossy().into_owned());
11835    }
11836
11837    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11838    // judge worktrees, so once the loop above has cleared all of them out,
11839    // the parent is a bare directory nobody else was ever going to remove -
11840    // git only ever managed what was inside it. Left alone, one of these
11841    // accumulates per fully-folded run; the operator's own machine had 74.
11842    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11843    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11844    // so does anything a run never claimed that happens to share the bay.
11845    remove_if_empty(&root);
11846
11847    if state.enabled_worktree_config && drop_winner {
11848        // A release, not a raw disable: some sibling run in this repository
11849        // may still hold its own reference (see `git::acquire_worktree_config`),
11850        // and only the last release actually turns the setting back off.
11851        git::release_worktree_config(&repo).await.ok();
11852        state.enabled_worktree_config = false;
11853    }
11854    state.save_under(home)?;
11855    Ok(removed)
11856}
11857
11858async fn verify_merged_fold_sources(
11859    state: &RunState,
11860    drop_winner: bool,
11861    home: &Path,
11862) -> Result<Vec<std::fs::File>> {
11863    let winner = state.tally.as_ref().map(|t| t.winner);
11864    let mut heads = BTreeSet::new();
11865    for candidate in &state.candidates {
11866        if Some(candidate.label) == winner && !drop_winner {
11867            continue;
11868        }
11869        anyhow::ensure!(
11870            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11871            "candidate recovery source is outside the inventoried run root: {}",
11872            candidate.worktree.display()
11873        );
11874        match std::fs::symlink_metadata(&candidate.worktree) {
11875            Ok(_) => {}
11876            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11877            Err(error) => return Err(error.into()),
11878        }
11879        if !state.released_branches.contains(&candidate.branch)
11880            && git::branch_exists(&state.repo, &candidate.branch).await?
11881        {
11882            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11883        }
11884    }
11885    let mut cargo_guards = Vec::new();
11886    let entries = match std::fs::read_dir(state.worktree_root()) {
11887        Ok(entries) => Some(entries),
11888        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11889        Err(error) => return Err(error.into()),
11890    };
11891    for entry in entries.into_iter().flatten() {
11892        let path = entry?.path();
11893        let metadata = std::fs::symlink_metadata(&path)?;
11894        anyhow::ensure!(
11895            metadata.is_dir() && !metadata.file_type().is_symlink(),
11896            "non-directory or symlink recovery source retained: {}",
11897            path.display()
11898        );
11899        if !drop_winner
11900            && winner.is_some_and(|w| {
11901                path.file_name()
11902                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11903            })
11904        {
11905            continue;
11906        }
11907        anyhow::ensure!(
11908            path.join(".git").is_file(),
11909            "unregistered recovery source retained: {}",
11910            path.display()
11911        );
11912        anyhow::ensure!(
11913            git::git(
11914                &path,
11915                &[
11916                    "status",
11917                    "--porcelain",
11918                    "--untracked-files=all",
11919                    "--ignored",
11920                    "--",
11921                    ".",
11922                    ":(exclude)target"
11923                ]
11924            )
11925            .await?
11926            .is_empty(),
11927            "source changes or ignored recovery files retained: {}",
11928            path.display()
11929        );
11930        match std::fs::symlink_metadata(path.join("target")) {
11931            Ok(_) => {
11932                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11933            }
11934            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11935            Err(error) => return Err(error.into()),
11936        }
11937        for checkpoint in [
11938            "rebase-merge",
11939            "rebase-apply",
11940            "sequencer",
11941            "CHERRY_PICK_HEAD",
11942            "MERGE_HEAD",
11943            "REBASE_HEAD",
11944            "REVERT_HEAD",
11945            "BISECT_LOG",
11946            "AUTO_MERGE",
11947            "MERGE_MSG",
11948        ] {
11949            let location = git::git(
11950                &path,
11951                &[
11952                    "rev-parse",
11953                    "--path-format=absolute",
11954                    "--git-path",
11955                    checkpoint,
11956                ],
11957            )
11958            .await?;
11959            anyhow::ensure!(
11960                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11961                if e.kind() == std::io::ErrorKind::NotFound),
11962                "Git recovery checkpoint retained: {checkpoint}"
11963            );
11964        }
11965        heads.insert(git::rev_parse(&path, "HEAD").await?);
11966    }
11967    crate::fold_evidence::preserve(state, home, &heads).await?;
11968
11969    Ok(cargo_guards)
11970}
11971
11972/// Remove `dir` if it exists and has nothing in it.
11973///
11974/// Best-effort and silent by design: a directory that is not empty (a run
11975/// whose winner is still parked there, a stray file some other process left)
11976/// is exactly the case this must refuse, and a directory that is already gone
11977/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11978/// already refuses a non-empty directory, so the emptiness check below is
11979/// belt, not suspenders - it is what keeps this from ever attempting the
11980/// removal in the case that matters, rather than trusting `remove_dir`'s
11981/// error path to have no side effects if it ever changed.
11982fn remove_if_empty(dir: &Path) {
11983    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11984        std::fs::remove_dir(dir).ok();
11985    }
11986}
11987
11988/// Severity of the worst open finding in the last review round, for reporting.
11989pub fn worst_open(state: &RunState) -> Option<Severity> {
11990    state
11991        .reviews
11992        .last()?
11993        .reviews
11994        .iter()
11995        .flat_map(|r| r.findings.iter())
11996        .map(|f| f.severity)
11997        .max()
11998}
11999
12000/// `[worktree] setup` hides its products per worktree, which needs
12001/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12002/// worktrees, like the hook's; every place that runs setup calls this first.
12003async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12004    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12005        git::acquire_worktree_config(repo).await?;
12006        state.enabled_worktree_config = true;
12007    }
12008    Ok(())
12009}
12010
12011#[cfg(test)]
12012mod tests {
12013    #[tokio::test]
12014    async fn launcher_base_fetch_requires_remote_success() {
12015        let tmp = tempfile::tempdir().unwrap();
12016        let repo = tmp.path().join("repo");
12017        let remote = tmp.path().join("remote");
12018        std::fs::create_dir(&repo).unwrap();
12019        std::fs::create_dir(&remote).unwrap();
12020        git::git(&remote, &["init", "--bare", "-b", "main"])
12021            .await
12022            .unwrap();
12023        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12024        git::git(
12025            &repo,
12026            &[
12027                "-c",
12028                "user.name=Fixture",
12029                "-c",
12030                "user.email=fixture@example.invalid",
12031                "commit",
12032                "--allow-empty",
12033                "-m",
12034                "fixture",
12035            ],
12036        )
12037        .await
12038        .unwrap();
12039        git::git(
12040            &repo,
12041            &["remote", "add", "origin", remote.to_str().unwrap()],
12042        )
12043        .await
12044        .unwrap();
12045        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12046        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12047        assert_eq!(
12048            resolve_base(&repo, "main", "origin", None)
12049                .await
12050                .unwrap()
12051                .commit,
12052            head
12053        );
12054        #[cfg(unix)]
12055        {
12056            let policy: crate::config::Coordination = toml::from_str(
12057                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12058            ).unwrap();
12059            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12060            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12061            let held = crate::repo_lock::SeatLease::operation_with(
12062                &repo,
12063                Some(&policy),
12064                &crate::repo_lock::WaitPolicy::default(),
12065            )
12066            .await
12067            .unwrap();
12068            assert!(held.is_some());
12069            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12070            tokio::pin!(fetch);
12071            assert!(
12072                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12073                    .await
12074                    .is_err()
12075            );
12076            assert!(
12077                !repo.join(".git/FETCH_HEAD").exists(),
12078                "no fetch before admission"
12079            );
12080            drop(held);
12081            let fetched = fetch.await.unwrap();
12082            assert_eq!(fetched.commit, head);
12083            assert_eq!(fetched.remote, "origin");
12084            assert_eq!(fetched.branch, "main");
12085        }
12086        // A surviving tracking ref is not successful fetch evidence.
12087        git::git(
12088            &repo,
12089            &[
12090                "remote",
12091                "set-url",
12092                "origin",
12093                tmp.path().join("missing").to_str().unwrap(),
12094            ],
12095        )
12096        .await
12097        .unwrap();
12098        let error = resolve_base(&repo, "main", "origin", None)
12099            .await
12100            .unwrap_err()
12101            .to_string();
12102        assert!(error.contains("cannot read origin/main"));
12103        assert!(error.contains("refusing to branch off"));
12104    }
12105
12106    #[test]
12107    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12108        let leaked = vec!["chore(deps): update a crate".to_owned()];
12109        let own = "fix(daemon): apply a chosen action";
12110        assert!(should_retitle("", own, &leaked));
12111        assert!(should_retitle(
12112            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12113            own,
12114            &leaked
12115        ));
12116        assert!(should_retitle(
12117            "chore: land candidate A of run 1",
12118            own,
12119            &leaked
12120        ));
12121        assert!(should_retitle(
12122            "magi: candidate A (uncommitted work)",
12123            own,
12124            &leaked
12125        ));
12126        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12127        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12128        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12129    }
12130
12131    #[test]
12132    fn pr_merge_plan_creates_adopts_or_stops() {
12133        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12134        assert_eq!(
12135            pr_merge_plan(Ok(land::OpenPr::One {
12136                url: "u".into(),
12137                title: "t".into()
12138            })),
12139            PrPlan::Adopt {
12140                url: "u".into(),
12141                title: "t".into()
12142            }
12143        );
12144        let PrPlan::Stop(many) =
12145            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12146        else {
12147            panic!("many must stop");
12148        };
12149        assert!(many.contains('a') && many.contains('b'));
12150        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12151            panic!("a failed lookup must stop");
12152        };
12153        assert!(err.contains("bad token"));
12154    }
12155
12156    use super::*;
12157    use crate::run::GateStatus;
12158    use std::collections::BTreeMap;
12159    use std::time::Duration;
12160
12161    fn conductor() -> AgentSpec {
12162        AgentSpec {
12163            id: "conductor".to_owned(),
12164            kind: crate::config::AgentKind::Command,
12165            model: None,
12166            command: vec!["true".to_owned()],
12167            extra_args: Vec::new(),
12168            env: BTreeMap::new(),
12169            prompt_delivery: None,
12170            max_turn_secs: None,
12171        }
12172    }
12173
12174    fn spec(id: &str) -> AgentSpec {
12175        AgentSpec {
12176            id: id.to_owned(),
12177            kind: crate::config::AgentKind::Command,
12178            model: None,
12179            command: vec!["true".to_owned()],
12180            extra_args: Vec::new(),
12181            env: BTreeMap::new(),
12182            prompt_delivery: None,
12183            max_turn_secs: None,
12184        }
12185    }
12186
12187    fn ids(xs: &[&str]) -> BTreeSet<String> {
12188        xs.iter().map(|s| (*s).to_owned()).collect()
12189    }
12190
12191    #[test]
12192    fn pick_successor_skips_an_agent_another_seat_holds() {
12193        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12194        let roster = [spec("a"), spec("b"), spec("c")];
12195        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12196        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12197    }
12198
12199    #[test]
12200    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12201        // The other seat started on c but was handed to d; c is free again.
12202        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12203        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12204        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12205        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12206        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12207    }
12208
12209    #[test]
12210    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12211        let roster = [spec("a"), spec("b")];
12212        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12213        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12214        let carried = ids(&["b"]);
12215        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12216        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12217    }
12218
12219    #[test]
12220    fn pick_successor_returns_none_without_any_untried_successor() {
12221        let roster = [spec("a"), spec("b")];
12222        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12223        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12224        let carried = ids(&["a"]);
12225        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12226    }
12227
12228    #[test]
12229    fn pick_successor_rescue_avoids_another_seats_occupant() {
12230        // b is a carried failure and free; a is held by the other seat.
12231        let roster = [spec("a"), spec("b"), spec("c")];
12232        let carried = ids(&["b", "c"]);
12233        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12234        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12235    }
12236
12237    #[test]
12238    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12239        let roster = [spec("a"), spec("b"), spec("c")];
12240        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12241        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12242    }
12243
12244    #[test]
12245    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12246        let roster = [spec("a"), spec("b"), spec("c")];
12247        let failed = ids(&["a", "b", "c"]);
12248        // Rescue looks at the whole roster, once per id, then runs out.
12249        let mut tried = ids(&["b"]);
12250        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12251        assert_eq!(first.id, "a");
12252        tried.insert(first.id.clone());
12253        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12254        assert_eq!(second.id, "c");
12255        tried.insert(second.id.clone());
12256        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12257    }
12258
12259    #[test]
12260    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12261        let roster = [spec("a"), spec("b")];
12262        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12263        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12264    }
12265
12266    #[test]
12267    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12268        let roster = [spec("a"), spec("b"), spec("c")];
12269        let h = SeatHistory {
12270            failed: ids(&["a"]),
12271            last_ok: Some("b".to_owned()),
12272            last_fail: None,
12273        };
12274        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12275        // A last answerer that left the roster, or later failed, is ignored.
12276        let gone = SeatHistory {
12277            last_ok: Some("zzz".to_owned()),
12278            ..h.clone()
12279        };
12280        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12281        let failed = SeatHistory {
12282            failed: ids(&["a", "b"]),
12283            last_ok: Some("b".to_owned()),
12284            last_fail: None,
12285        };
12286        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12287    }
12288
12289    #[test]
12290    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12291        let first = SeatState::new("review-1", "alpha", 7);
12292        let next = handover_seat("review-1", "gamma", 7);
12293        assert_ne!(first.claude_session, next.claude_session);
12294    }
12295
12296    #[test]
12297    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12298        let mut state = state_with_summary("x", "y");
12299        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12300        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12301        assert_ne!(a.claude_session, b.claude_session);
12302    }
12303
12304    #[test]
12305    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12306        let roster = [spec("a"), spec("b")];
12307        let h = SeatHistory {
12308            failed: ids(&["a", "b"]),
12309            ..SeatHistory::default()
12310        };
12311        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12312        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12313        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12314    }
12315
12316    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12317    // fallback loop depends on to terminate: it must walk forward from the
12318    // seat's own position, never restart at the front of the roster, and it
12319    // must never hand back an id already tried, however many times that id
12320    // happens to appear.
12321
12322    #[test]
12323    fn failure_signature_ignores_numbers_and_paths() {
12324        assert_eq!(
12325            failure_signature("exited with Some(2) and no usable output"),
12326            failure_signature("exited with Some(137) and no usable output")
12327        );
12328        assert_eq!(
12329            failure_signature("process exited with no usable output (exit code 2)"),
12330            failure_signature("process exited with no usable output (exit code 137)")
12331        );
12332        for (old, new) in [
12333            (
12334                "exited with Some(2) and no usable output",
12335                "process exited with no usable output (exit code 2)",
12336            ),
12337            (
12338                "exited with None and no usable output",
12339                "process exited with no usable output (exit code unavailable)",
12340            ),
12341        ] {
12342            let previous = FailClass::Other(failure_signature(old));
12343            let current = FailClass::Other(failure_signature(new));
12344            assert_eq!(previous, current, "resumption retains the failure class");
12345            assert!(!should_hand_over(Some(&previous), &current));
12346        }
12347        assert_eq!(
12348            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12349            failure_signature("cannot open /var/x.txt: denied")
12350        );
12351        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12352    }
12353
12354    #[test]
12355    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12356        let other = FailClass::Other("x".into());
12357        assert!(should_hand_over(None, &FailClass::Quota));
12358        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12359        assert!(should_hand_over(
12360            Some(&FailClass::Timeout),
12361            &FailClass::Timeout
12362        ));
12363        assert!(should_hand_over(None, &other));
12364        assert!(!should_hand_over(Some(&other), &other));
12365        assert!(should_hand_over(
12366            Some(&other),
12367            &FailClass::Other("y".into())
12368        ));
12369        // A quota or timeout in between ends the run of identical failures.
12370        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12371        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12372    }
12373
12374    #[test]
12375    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12376        let a = SeatState::new("judge-1", "alpha", 7);
12377        let b = handover_seat("judge-1", "beta", 7);
12378        assert_ne!(a.claude_session, b.claude_session);
12379        assert_eq!(b.turns, 0);
12380    }
12381
12382    #[test]
12383    fn a_timeout_is_classified_apart_from_other_failures() {
12384        assert_eq!(
12385            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12386            Some(FailClass::Timeout)
12387        );
12388        assert!(matches!(
12389            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12390            Some(FailClass::Other(_))
12391        ));
12392    }
12393
12394    #[test]
12395    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12396        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12397        let tried = BTreeSet::from(["beta".to_owned()]);
12398        // beta sits at index 1; the next candidate is gamma, never alpha —
12399        // which is very likely a different candidate slot's own agent.
12400        let next = next_untried_in_roster(&roster, 1, &tried);
12401        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12402    }
12403
12404    #[test]
12405    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12406        let roster = vec![spec("alpha"), spec("beta")];
12407        let tried = BTreeSet::from(["beta".to_owned()]);
12408        // beta is the roster's last entry: nothing follows it, and alpha —
12409        // earlier in the roster, almost certainly a different candidate
12410        // slot's own agent — must not be reached by wrapping back to it.
12411        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12412    }
12413
12414    #[test]
12415    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12416        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12417        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12418        // beta (index 1) and gamma (index 2, the only entry after it) have
12419        // both been tried; alpha (index 0) never has, but it comes before
12420        // beta's own position, so there is nothing further for this seat.
12421        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12422    }
12423
12424    #[test]
12425    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12426        let roster = vec![spec("a"), spec("a"), spec("b")];
12427        let tried = BTreeSet::from(["a".to_owned()]);
12428        let next = next_untried_in_roster(&roster, 0, &tried);
12429        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12430    }
12431
12432    #[test]
12433    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12434        let roster = vec![spec("a"), spec("b")];
12435        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12436        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12437    }
12438
12439    #[test]
12440    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12441        let dir = tempfile::tempdir().unwrap();
12442        let bay = dir.path().join("ffff");
12443
12444        // Not there yet: nothing to do, nothing to panic on.
12445        remove_if_empty(&bay);
12446        assert!(!bay.exists());
12447
12448        // Something still inside - the winner's worktree, or a stray file -
12449        // keeps the directory standing.
12450        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12451        remove_if_empty(&bay);
12452        assert!(bay.exists(), "non-empty directory must survive");
12453
12454        // Once the last entry is gone, so is the directory itself.
12455        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12456        remove_if_empty(&bay);
12457        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12458    }
12459
12460    // `round_is_clean` is the exact decision this task fixed: a round with a
12461    // seat that never answered must not read the same as a round every seat
12462    // actually reviewed. These are deterministic and process-free by design —
12463    // the equivalent end-to-end check (a real reviewer timing out under a
12464    // live graph run) is a genuine race against wall-clock contention, and a
12465    // spawn slow enough to blow even a generous budget under a loaded test
12466    // run must not turn this specific regression check flaky.
12467
12468    #[test]
12469    fn a_full_panel_that_found_nothing_is_clean() {
12470        assert!(round_is_clean(
12471            0,
12472            true,
12473            2,
12474            2,
12475            0,
12476            IncompleteReviewPolicy::Block
12477        ));
12478    }
12479
12480    #[test]
12481    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12482        assert!(!round_is_clean(
12483            0,
12484            true,
12485            1,
12486            2,
12487            0,
12488            IncompleteReviewPolicy::Block
12489        ));
12490    }
12491
12492    #[test]
12493    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12494        assert!(!round_is_clean(
12495            1,
12496            true,
12497            1,
12498            2,
12499            0,
12500            IncompleteReviewPolicy::Warn
12501        ));
12502    }
12503
12504    #[test]
12505    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12506        assert!(round_is_clean(
12507            0,
12508            true,
12509            1,
12510            2,
12511            0,
12512            IncompleteReviewPolicy::Warn
12513        ));
12514    }
12515
12516    #[test]
12517    fn a_full_panel_with_an_open_finding_is_not_clean() {
12518        assert!(!round_is_clean(
12519            1,
12520            true,
12521            2,
12522            2,
12523            0,
12524            IncompleteReviewPolicy::Block
12525        ));
12526    }
12527
12528    #[test]
12529    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12530        assert!(!round_is_clean(
12531            0,
12532            false,
12533            2,
12534            2,
12535            0,
12536            IncompleteReviewPolicy::Block
12537        ));
12538    }
12539
12540    // The stall this task closes: under the default `block` policy, a seat
12541    // missing only because it was rate limited must not force a wait for a
12542    // session limit that will not lift by the next round. `round_is_clean`
12543    // is where that quorum carve-out lives; the review loop around it never
12544    // changes what a reviewer's vote or a finding's severity means.
12545
12546    #[test]
12547    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12548        // 1 of 2 answered, and the one missing was quota'd — the exact
12549        // "review-2 rate limited (quota)" shape from the field report.
12550        assert!(round_is_clean(
12551            0,
12552            true,
12553            1,
12554            2,
12555            1,
12556            IncompleteReviewPolicy::Block
12557        ));
12558    }
12559
12560    #[test]
12561    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12562        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12563        // not a quota loss (`quota_missing` stays 0) — worth another try.
12564        assert!(!round_is_clean(
12565            0,
12566            true,
12567            1,
12568            2,
12569            0,
12570            IncompleteReviewPolicy::Block
12571        ));
12572    }
12573
12574    #[test]
12575    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12576        assert!(!round_is_clean(
12577            1,
12578            true,
12579            1,
12580            2,
12581            1,
12582            IncompleteReviewPolicy::Block
12583        ));
12584        assert!(!round_is_clean(
12585            0,
12586            false,
12587            1,
12588            2,
12589            1,
12590            IncompleteReviewPolicy::Block
12591        ));
12592    }
12593
12594    #[test]
12595    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12596        // Every seat quota'd, nobody answered: there is no panel to decide
12597        // on, so this must fall through to the existing block-and-retry
12598        // fallback rather than call an unreviewed patch clean.
12599        assert!(!round_is_clean(
12600            0,
12601            true,
12602            0,
12603            2,
12604            2,
12605            IncompleteReviewPolicy::Block
12606        ));
12607    }
12608
12609    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12610        CommandOutcome {
12611            command: "test".to_owned(),
12612            code,
12613            output_tail: String::new(),
12614            duration_ms: 0,
12615            resource_blocked,
12616        }
12617    }
12618
12619    #[test]
12620    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12621        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12622        assert!(
12623            !verify_inconclusive(&[outcome(Some(1), false)]),
12624            "an ordinary failure is still evidence about the patch"
12625        );
12626        assert!(verify_inconclusive(&[outcome(None, true)]));
12627        assert!(
12628            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12629            "one inconclusive outcome taints the whole batch"
12630        );
12631        assert!(!verify_inconclusive(&[]));
12632    }
12633
12634    #[tokio::test]
12635    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12636        // Alive for the first two checks, then dead - confirms the loop
12637        // actually re-polls rather than deciding once and sleeping out the
12638        // ceiling regardless.
12639        let calls = std::sync::atomic::AtomicUsize::new(0);
12640        let started = Instant::now();
12641        wait_for_pids_with(
12642            &[123],
12643            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12644            Duration::from_millis(5),
12645            Duration::from_secs(5),
12646        )
12647        .await;
12648        assert!(
12649            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12650            "must keep checking rather than deciding on the first answer"
12651        );
12652        assert!(
12653            started.elapsed() < Duration::from_secs(1),
12654            "must return the moment it is confirmed dead, not wait out the ceiling"
12655        );
12656    }
12657
12658    #[tokio::test]
12659    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12660        let started = Instant::now();
12661        wait_for_pids_with(
12662            &[123],
12663            |_| true, // never reports dead
12664            Duration::from_millis(5),
12665            Duration::from_millis(30),
12666        )
12667        .await;
12668        let elapsed = started.elapsed();
12669        assert!(
12670            elapsed >= Duration::from_millis(30),
12671            "must not give up before its own ceiling: {elapsed:?}"
12672        );
12673        assert!(
12674            elapsed < Duration::from_secs(1),
12675            "must not wait past its own ceiling either: {elapsed:?}"
12676        );
12677    }
12678
12679    #[tokio::test]
12680    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12681        let started = Instant::now();
12682        wait_for_pids_with(
12683            &[],
12684            |_| true,
12685            Duration::from_secs(5),
12686            Duration::from_secs(5),
12687        )
12688        .await;
12689        assert!(
12690            started.elapsed() < Duration::from_millis(200),
12691            "an empty pid list has nothing to confirm"
12692        );
12693    }
12694
12695    // `review_conclusion` is the exact decision the review hand-off task
12696    // fixed: a round budget spent (or a tree that stopped moving) must not
12697    // collapse into `Blocked` regardless of what verification actually
12698    // said. Deterministic and process-free for the same reason the
12699    // `round_is_clean` family above is.
12700    fn review_round(
12701        clean: bool,
12702        blocking: usize,
12703        answered: usize,
12704        expected: usize,
12705        progressed: bool,
12706        e2e_ok: bool,
12707    ) -> ReviewRound {
12708        ReviewRound {
12709            round: 1,
12710            head: "h".to_owned(),
12711            verified_head: None,
12712            verified_at: None,
12713            reviews: Vec::new(),
12714            e2e: vec![CommandOutcome {
12715                command: "test".to_owned(),
12716                code: Some(if e2e_ok { 0 } else { 1 }),
12717                output_tail: String::new(),
12718                duration_ms: 0,
12719                resource_blocked: false,
12720            }],
12721            verify_retried: false,
12722            e2e_deferred: false,
12723            e2e_defer_reason: None,
12724            fix: None,
12725            blocking,
12726            answered,
12727            expected,
12728            clean,
12729            progressed,
12730            vote_split: false,
12731            reconsideration: Vec::new(),
12732            verdict: None,
12733        }
12734    }
12735
12736    #[test]
12737    fn review_conclusion_is_none_when_nothing_has_run() {
12738        assert_eq!(review_conclusion(&[], 3), None);
12739    }
12740
12741    #[test]
12742    fn review_conclusion_is_none_while_rounds_remain() {
12743        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12744        assert_eq!(review_conclusion(&rounds, 3), None);
12745    }
12746
12747    #[test]
12748    fn review_conclusion_is_gating_once_a_round_is_clean() {
12749        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12750        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12751    }
12752
12753    #[test]
12754    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12755        let rounds = vec![
12756            review_round(false, 1, 2, 2, true, true),
12757            review_round(false, 1, 2, 2, true, true),
12758        ];
12759        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12760    }
12761
12762    #[test]
12763    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12764        let rounds = vec![
12765            review_round(false, 1, 2, 2, true, true),
12766            review_round(false, 1, 2, 2, true, false),
12767        ];
12768        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12769    }
12770
12771    #[test]
12772    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12773        // Magi never got a command to run against this round's own head — a
12774        // resource-blocked attempt, not a red one — so this must never
12775        // settle on `Blocked` the way a genuine e2e failure would. `None`
12776        // here is what tells `Runner::review_loop` to retry the check
12777        // itself rather than trust this cheap recomputation with a verdict
12778        // it cannot actually produce.
12779        let mut blocked = review_round(false, 1, 2, 2, true, false);
12780        blocked.e2e[0].resource_blocked = true;
12781        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12782        assert_eq!(review_conclusion(&rounds, 2), None);
12783    }
12784
12785    #[test]
12786    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12787        // Missing input, not a verified tree — never a hand-off candidate.
12788        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12789        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12790    }
12791
12792    #[test]
12793    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12794        let rounds = vec![
12795            review_round(false, 1, 2, 2, false, true),
12796            review_round(false, 1, 2, 2, false, true),
12797        ];
12798        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12799    }
12800
12801    fn secs(n: u64) -> Duration {
12802        Duration::from_secs(n)
12803    }
12804
12805    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12806    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12807    #[tokio::test]
12808    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12809        ask_test_home();
12810        let tmp = tempfile::tempdir().unwrap();
12811        let repo = tmp.path().join("repo");
12812        std::fs::create_dir_all(&repo).unwrap();
12813        init_repo(&repo);
12814        let g = |args: &[&str]| {
12815            let out = std::process::Command::new("git")
12816                .args(args)
12817                .current_dir(&repo)
12818                .quiet()
12819                .output()
12820                .unwrap();
12821            assert!(
12822                out.status.success(),
12823                "{}",
12824                String::from_utf8_lossy(&out.stderr)
12825            );
12826        };
12827        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12828        g(&["checkout", "-b", "historical"]);
12829        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12830        g(&["add", "."]);
12831        g(&["commit", "-m", "old alternative"]);
12832        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12833        g(&["checkout", "-b", "magi/task-selected", &base]);
12834        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12835        g(&["add", "."]);
12836        g(&["commit", "-m", "measurement infrastructure"]);
12837        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12838        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12839        g(&["add", "."]);
12840        g(&["commit", "-m", "dependent implementation"]);
12841        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12842        g(&["checkout", "main"]);
12843        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12844        g(&["add", "."]);
12845        g(&["commit", "-m", "unrelated base change"]);
12846        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12847        let origin = tmp.path().join("origin.git");
12848        g(&[
12849            "clone",
12850            "--bare",
12851            repo.to_str().unwrap(),
12852            origin.to_str().unwrap(),
12853        ]);
12854        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12855        g(&["fetch", "origin"]);
12856        let selection = crate::pr_recovery::Selection {
12857            url: "https://example.test/project/pull/1".into(),
12858            branch: "magi/task-selected".into(),
12859            head: head.clone(),
12860            source_head: None,
12861            source_run: None,
12862            source_proof: None,
12863            disposition: Some(crate::pr_recovery::Disposition::Review),
12864            instruction: "preserve measured behavior".into(),
12865            approved_at: jiff::Timestamp::now(),
12866        };
12867        let mut template = runner_at(RunStatus::Reviewing);
12868        template.state.repo = repo.clone();
12869        template.state.base_commit = current.clone();
12870        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12871        template.state.origin = Some(crate::run::Origin::queue("selected"));
12872        template.state.pr_head = Some(selection.branch.clone());
12873        template.state.pr_recovery = Some(selection.clone());
12874        let mut runner =
12875            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12876                .await
12877                .unwrap();
12878        let wt = runner.state.winner().unwrap().worktree.clone();
12879        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12880        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12881        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12882        assert!(runner.state.seeds.is_empty());
12883        assert_eq!(runner.state.instruction, "preserve measured behavior");
12884        runner.sync_to_base().await.unwrap();
12885        assert_eq!(
12886            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12887            "prerequisite"
12888        );
12889        assert_eq!(
12890            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12891            "measured implementation"
12892        );
12893        assert_eq!(
12894            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12895            "current main change"
12896        );
12897        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12898        assert_eq!(
12899            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12900            head
12901        );
12902
12903        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12904        let mut partial = runner_at(RunStatus::Reviewing);
12905        partial.state.repo = repo.clone();
12906        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12907        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12908            run: Some("exit 1".into()),
12909            ..Default::default()
12910        }];
12911        let selection = runner.state.pr_recovery.clone().unwrap();
12912        partial.state.pr_recovery = Some(selection.clone());
12913        partial.state.pr_head = Some(selection.branch.clone());
12914        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12915        let id = partial.state.id.clone();
12916        let error = match Runner::open_selected(
12917            &repo,
12918            partial.state,
12919            partial.roles,
12920            1,
12921            runner.state.base_commit.clone(),
12922            selection,
12923        )
12924        .await
12925        {
12926            Ok(_) => panic!("setup must fail"),
12927            Err(e) => e,
12928        };
12929        assert_eq!(
12930            error
12931                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12932                .unwrap()
12933                .run,
12934            id
12935        );
12936        let failed = RunState::load(&id).unwrap();
12937        assert_eq!(failed.status, RunStatus::Blocked);
12938        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12939        assert!(Runner::resume(&id).is_err());
12940        runner.state.status = RunStatus::Ready;
12941        runner.state.driver_exited = true;
12942        runner.state.save().unwrap();
12943        let mut retry = runner_at(RunStatus::Reviewing);
12944        retry.state.repo = repo.clone();
12945        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12946        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12947        reselected.approved_at = jiff::Timestamp::now();
12948        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12949        retry.state.pr_head = Some(reselected.branch.clone());
12950        retry.state.pr_recovery = Some(reselected.clone());
12951        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12952            .await
12953            .unwrap();
12954        let restarted = Runner::open_selected(
12955            &repo,
12956            retry.state,
12957            retry.roles,
12958            1,
12959            runner.state.base_commit.clone(),
12960            reselected,
12961        )
12962        .await
12963        .unwrap();
12964        assert_eq!(
12965            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12966                .await
12967                .unwrap(),
12968            head
12969        );
12970        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12971        assert_eq!(
12972            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12973            head
12974        );
12975    }
12976
12977    #[tokio::test]
12978    #[cfg(unix)]
12979    async fn fallback_and_interrupted_restoration_require_coordination() {
12980        use std::os::fd::AsRawFd as _;
12981        ask_test_home();
12982        let tmp = tempfile::tempdir().unwrap();
12983        let mut runner = registered_runner(tmp.path()).await;
12984        runner.state.config.graph.timeout_resource_wait = 1;
12985        let winner = runner.state.candidates[0].clone();
12986        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12987        let leftover = winner.worktree.join("leftover");
12988        std::fs::write(&leftover, "untrusted edits").unwrap();
12989        let next = tmp.path().join("next-fixer");
12990        let mut failed = spec("first");
12991        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12992        let mut successor = spec("second");
12993        successor.command = vec![
12994            "sh".into(),
12995            "-c".into(),
12996            format!("touch '{}'; echo done", next.display()),
12997        ];
12998        let attempts = vec![
12999            (failed.clone(), "first".into()),
13000            (successor.clone(), "second".into()),
13001        ];
13002        let lock_path = runner.state.repo.join(".git/updater.lock");
13003        let external = std::fs::File::create(&lock_path).unwrap();
13004        assert_eq!(
13005            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13006            0
13007        );
13008        let artifacts = tmp.path().join("artifacts");
13009        let build = |spec, seat| SeatJob {
13010            spec,
13011            seat,
13012            cwd: winner.worktree.clone(),
13013            prompt: "fixture".into(),
13014            timeout: Duration::from_secs(10),
13015            allow_write: true,
13016            sessions: false,
13017            artifacts: artifacts.clone(),
13018            stem: "fix".into(),
13019            handover: None,
13020        };
13021        let (_, _, out) = runner
13022            .ask_fixer_among(
13023                attempts.clone(),
13024                "graft",
13025                None,
13026                Some((&winner.worktree, &head)),
13027                build,
13028            )
13029            .await;
13030        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13031        assert!(leftover.exists(), "contention must prevent clean");
13032        assert!(!next.exists());
13033        assert!(runner.state.handovers.is_empty());
13034        // Early interrupted-repair rollback uses the same protected route.
13035        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13036            from_run: "earlier".into(),
13037            generation: 1,
13038            mode: crate::recovery::Mode::VerifyRepair,
13039            reasons: Vec::new(),
13040            reviewed_head: None,
13041            rounds: Vec::new(),
13042            open_findings: Vec::new(),
13043            evidence: None,
13044            criteria_inherited: true,
13045            repair: crate::recovery::Repair::Started {
13046                before: head.clone(),
13047            },
13048        });
13049        assert!(
13050            runner
13051                .recovery_repair(&winner)
13052                .await
13053                .unwrap_err()
13054                .is::<crate::repo_lock::ResourceBlocked>()
13055        );
13056        assert!(matches!(
13057            runner.state.recovery.as_ref().unwrap().repair,
13058            crate::recovery::Repair::Started { .. }
13059        ));
13060        assert!(leftover.exists());
13061        drop(external);
13062        runner.recovery_repair(&winner).await.unwrap();
13063        assert!(!leftover.exists());
13064        assert!(matches!(
13065            runner.state.recovery.as_ref().unwrap().repair,
13066            crate::recovery::Repair::Done { .. }
13067        ));
13068        // A hard lock-open failure also leaves HEAD and untracked work intact.
13069        std::fs::remove_file(&lock_path).unwrap();
13070        std::fs::create_dir(&lock_path).unwrap();
13071        std::fs::write(&leftover, "untrusted edits").unwrap();
13072        let (_, _, out) = runner
13073            .ask_fixer_among(
13074                attempts,
13075                "graft",
13076                None,
13077                Some((&winner.worktree, &head)),
13078                build,
13079            )
13080            .await;
13081        assert!(matches!(out, AgentOutcome::Failed(_)));
13082        assert!(leftover.exists());
13083        assert!(!next.exists());
13084        assert_eq!(
13085            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13086            head
13087        );
13088    }
13089
13090    #[cfg(unix)]
13091    async fn registered_runner(root: &Path) -> Runner {
13092        ask_test_home();
13093        let repo = root.join("repo");
13094        std::fs::create_dir_all(&repo).unwrap();
13095        init_repo(&repo);
13096        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13097        let mut runner = runner_at(RunStatus::Prep);
13098        runner.state.repo = repo.clone();
13099        runner.state.config.coordination = crate::config::test_coordination();
13100        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13101        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13102        runner.state.config.graph.advise = false;
13103        runner.roles.implementers = vec![spec("implement")];
13104        runner.roles.judges = vec![spec("judge")];
13105        runner.prep().await.unwrap();
13106        runner
13107    }
13108
13109    #[tokio::test]
13110    #[cfg(unix)]
13111    async fn parked_implementer_persists_completed_work() {
13112        for registered in [false, true] {
13113            let tmp = tempfile::tempdir().unwrap();
13114            let mut runner = registered_runner(tmp.path()).await;
13115            if !registered {
13116                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13117            }
13118            let marker = tmp.path().join("started");
13119            let release = tmp.path().join("release");
13120            let mut implementer = spec("implement");
13121            implementer.command = vec![
13122                "sh".into(),
13123                "-c".into(),
13124                format!(
13125                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13126                    marker.display(),
13127                    release.display()
13128                ),
13129            ];
13130            runner.roles.implementers = vec![implementer.clone()];
13131            runner.state.config.agents = vec![implementer];
13132            let pause = runner.pause.clone();
13133            let (result, ()) = tokio::join!(runner.implement(), async {
13134                tokio::time::timeout(Duration::from_secs(10), async {
13135                    while !marker.exists() {
13136                        tokio::time::sleep(Duration::from_millis(10)).await;
13137                    }
13138                })
13139                .await
13140                .unwrap();
13141                pause.park();
13142                std::fs::write(&release, "go").unwrap();
13143            });
13144            result.unwrap();
13145            let candidate = &runner.state.candidates[0];
13146            assert!(candidate.commits > 0, "{candidate:?}");
13147            assert!(candidate.summary.contains("saved-result"));
13148            assert_eq!(candidate.files, 1);
13149            let before = candidate.commits;
13150            runner.implement().await.unwrap();
13151            assert_eq!(runner.state.candidates[0].commits, before);
13152        }
13153    }
13154
13155    #[tokio::test]
13156    #[cfg(unix)]
13157    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13158        let tmp = tempfile::tempdir().unwrap();
13159        let mut runner = registered_runner(tmp.path()).await;
13160        let winner = runner.state.candidates[0].clone();
13161        let cache = tmp.path().join("cache");
13162        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13163        runner.state.config.graph.timeout_resource_wait = 0;
13164        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13165        let crate::cache::AcquireOutcome::Acquired(held) =
13166            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13167        else {
13168            panic!("cache must be free")
13169        };
13170        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13171            from_run: "earlier".into(),
13172            generation: 1,
13173            mode: crate::recovery::Mode::VerifyRepair,
13174            reasons: Vec::new(),
13175            reviewed_head: None,
13176            rounds: Vec::new(),
13177            open_findings: Vec::new(),
13178            evidence: None,
13179            criteria_inherited: true,
13180            repair: crate::recovery::Repair::NotStarted,
13181        });
13182        runner.recovery_repair(&winner).await.unwrap();
13183        assert!(matches!(
13184            runner.state.recovery.as_ref().unwrap().repair,
13185            crate::recovery::Repair::WaitingForRestore { .. }
13186        ));
13187        drop(held);
13188        runner.state.resource_block = None;
13189        runner.state.config.graph.timeout_resource_wait = 1;
13190        runner.recovery_repair(&winner).await.unwrap();
13191        assert!(matches!(
13192            runner.state.recovery.as_ref().unwrap().repair,
13193            crate::recovery::Repair::Done { .. }
13194        ));
13195        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13196    }
13197
13198    #[tokio::test]
13199    #[cfg(unix)]
13200    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13201        use std::os::fd::AsRawFd as _;
13202        for graft in [false, true] {
13203            let tmp = tempfile::tempdir().unwrap();
13204            let mut runner = registered_runner(tmp.path()).await;
13205            runner.state.config.graph.timeout_resource_wait = 1;
13206            let winner = runner.state.candidates[0].clone();
13207            let marker = tmp.path().join("fixer-started");
13208            let mut fixer = spec("fixer");
13209            fixer.command = vec![
13210                "sh".into(),
13211                "-c".into(),
13212                format!("echo started >> '{}'; echo done", marker.display()),
13213            ];
13214            runner.roles.fixer = Some(vec![fixer]);
13215            if graft {
13216                let mut loser = winner.clone();
13217                loser.label = 'B';
13218                loser.index = 1;
13219                runner.state.candidates.push(loser);
13220                runner.state.config.graph.graft_runner_up = true;
13221                runner.state.tally = Some(Tally {
13222                    first_choice: BTreeMap::new(),
13223                    borda: BTreeMap::new(),
13224                    winner: winner.label,
13225                    rankings: 1,
13226                    unanimous_initial: true,
13227                    deliberated: false,
13228                    changed_votes: 0,
13229                    unanimous_final: true,
13230                    tie_break: None,
13231                    judges: 1,
13232                    present: 1,
13233                    quorum: 1,
13234                    met_quorum: true,
13235                    uncontested: None,
13236                });
13237                runner.state.judgements.push(crate::run::Judgement {
13238                    judge: 1,
13239                    seat: "judge-1".into(),
13240                    agent: "judge".into(),
13241                    ranking: vec![winner.label, 'B'],
13242                    reasons: BTreeMap::new(),
13243                    confidence: None,
13244                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13245                    order: vec![0, 1],
13246                    failed: None,
13247                    duration_ms: 0,
13248                });
13249            } else {
13250                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13251                    from_run: "earlier".into(),
13252                    generation: 1,
13253                    mode: crate::recovery::Mode::VerifyRepair,
13254                    reasons: Vec::new(),
13255                    reviewed_head: None,
13256                    rounds: Vec::new(),
13257                    open_findings: Vec::new(),
13258                    evidence: None,
13259                    criteria_inherited: true,
13260                    repair: crate::recovery::Repair::NotStarted,
13261                });
13262            }
13263            let reader = crate::repo_lock::SeatLease::for_checkout(
13264                &winner.worktree,
13265                false,
13266                crate::config::test_coordination().as_ref(),
13267            )
13268            .await
13269            .unwrap();
13270            let pause = runner.pause.clone();
13271            let (result, ()) = tokio::join!(
13272                async {
13273                    if graft {
13274                        runner.graft().await
13275                    } else {
13276                        runner.recovery_repair(&winner).await
13277                    }
13278                },
13279                async {
13280                    tokio::time::sleep(Duration::from_millis(100)).await;
13281                    pause.park();
13282                }
13283            );
13284            result.unwrap();
13285            assert!(!marker.exists(), "a blocked seat must not spawn");
13286            // Exercise the persisted state, as a binary replacement/resume does.
13287            runner.state = RunState::load(&runner.state.id).unwrap();
13288            if graft {
13289                assert_eq!(
13290                    runner.state.graft.as_ref().unwrap().outcome,
13291                    GraftOutcome::WaitingForRestore
13292                );
13293            } else {
13294                assert!(matches!(
13295                    runner.state.recovery.as_ref().unwrap().repair,
13296                    crate::recovery::Repair::WaitingForRestore { .. }
13297                ));
13298            }
13299            drop(reader);
13300            runner.pause = Pause::new();
13301            runner.state.resource_block = None;
13302            // Deferred restoration must still respect an external lock owner.
13303            let external =
13304                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13305            assert_eq!(
13306                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13307                0
13308            );
13309            let leftover = winner.worktree.join("leftover");
13310            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13311            let result = if graft {
13312                runner.graft().await
13313            } else {
13314                runner.recovery_repair(&winner).await
13315            };
13316            assert!(
13317                result
13318                    .unwrap_err()
13319                    .is::<crate::repo_lock::ResourceBlocked>()
13320            );
13321            assert!(leftover.exists());
13322            assert!(!marker.exists());
13323            drop(external);
13324            runner.state.resource_block = None;
13325            let result = if graft {
13326                runner.graft().await
13327            } else {
13328                runner.recovery_repair(&winner).await
13329            };
13330            result.unwrap();
13331            assert!(!leftover.exists(), "restore must precede retry");
13332            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13333            if graft {
13334                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13335            } else {
13336                assert!(matches!(
13337                    runner.state.recovery.as_ref().unwrap().repair,
13338                    crate::recovery::Repair::Done { .. }
13339                ));
13340            }
13341        }
13342    }
13343
13344    #[tokio::test]
13345    #[cfg(unix)]
13346    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13347        use std::os::fd::AsRawFd as _;
13348        let tmp = tempfile::tempdir().unwrap();
13349        let mut runner = registered_runner(tmp.path()).await;
13350        runner.state.config.graph.timeout_resource_wait = 1;
13351        let winner = runner.state.candidates[0].clone();
13352        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13353        runner.state.tally = Some(Tally {
13354            first_choice: BTreeMap::new(),
13355            borda: BTreeMap::new(),
13356            winner: winner.label,
13357            rankings: 0,
13358            unanimous_initial: false,
13359            deliberated: false,
13360            changed_votes: 0,
13361            unanimous_final: false,
13362            tie_break: None,
13363            judges: 0,
13364            present: 0,
13365            quorum: 0,
13366            met_quorum: true,
13367            uncontested: Some("fixture".into()),
13368        });
13369        runner.state.graft = Some(GraftRecord {
13370            pre: pre.clone(),
13371            post: None,
13372            lines: None,
13373            limit: None,
13374            agent: None,
13375            outcome: GraftOutcome::Running,
13376            requested: Vec::new(),
13377            adoption: None,
13378            adoption_error: None,
13379        });
13380        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13381        git::commit_all(&winner.worktree, "unfinished graft")
13382            .await
13383            .unwrap();
13384        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13385        let leftover = winner.worktree.join("leftover");
13386        std::fs::write(&leftover, "untrusted edits").unwrap();
13387        let path = runner.state.repo.join(".git/updater.lock");
13388        let external = std::fs::File::create(&path).unwrap();
13389        assert_eq!(
13390            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13391            0
13392        );
13393        assert!(
13394            runner
13395                .graft()
13396                .await
13397                .unwrap_err()
13398                .is::<crate::repo_lock::ResourceBlocked>()
13399        );
13400        assert!(matches!(
13401            runner.state.graft.as_ref().unwrap().outcome,
13402            GraftOutcome::Running
13403        ));
13404        assert_eq!(
13405            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13406            unfinished
13407        );
13408        assert!(leftover.exists());
13409        drop(external);
13410        std::fs::remove_file(&path).unwrap();
13411        std::fs::create_dir(&path).unwrap();
13412        assert!(runner.graft().await.is_err());
13413        assert!(matches!(
13414            runner.state.graft.as_ref().unwrap().outcome,
13415            GraftOutcome::Running
13416        ));
13417        assert_eq!(
13418            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13419            unfinished
13420        );
13421        assert!(leftover.exists());
13422        std::fs::remove_dir(&path).unwrap();
13423        runner.graft().await.unwrap();
13424        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13425        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13426        assert!(!leftover.exists());
13427    }
13428
13429    #[tokio::test]
13430    #[cfg(unix)]
13431    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13432        use std::os::fd::AsRawFd as _;
13433        let tmp = tempfile::tempdir().unwrap();
13434        let mut runner = registered_runner(tmp.path()).await;
13435        runner.state.prep_complete = Some(false);
13436        runner.state.config.graph.timeout_resource_wait = 1;
13437        runner.state.save().unwrap();
13438        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13439        assert_eq!(
13440            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13441            0
13442        );
13443        runner.execute().await.unwrap();
13444        let disk = RunState::load(&runner.state.id).unwrap();
13445        assert!(disk.driver_exited);
13446        assert_eq!(disk.driver_pid, None);
13447        assert_eq!(runner.state.status, RunStatus::Blocked);
13448        assert_eq!(
13449            runner.state.resource_block.as_ref().unwrap().kind,
13450            "repository coordination"
13451        );
13452        assert!(runner.state.prep_failure.is_none());
13453        assert!(runner.state.seats.is_empty());
13454        runner.pause.park();
13455        // Include headroom for Git subprocesses under parallel test load;
13456        // the repository contention deadline itself remains one second.
13457        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13458            .await
13459            .unwrap()
13460            .unwrap();
13461        assert!(runner.state.parked);
13462        assert!(runner.state.prep_failure.is_none());
13463        let disk = RunState::load(&runner.state.id).unwrap();
13464        assert!(disk.driver_exited);
13465        assert_eq!(disk.driver_pid, None);
13466    }
13467
13468    #[tokio::test]
13469    #[cfg(unix)]
13470    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13471        use std::os::fd::AsRawFd as _;
13472        let tmp = tempfile::tempdir().unwrap();
13473        let mut runner = registered_runner(tmp.path()).await;
13474        runner.state.prep_complete = Some(false);
13475        runner.state.config.graph.timeout_resource_wait = 30;
13476        runner.state.save().unwrap();
13477        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13478        assert_eq!(
13479            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13480            0
13481        );
13482        assert!(
13483            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13484                .await
13485                .is_err()
13486        );
13487        let disk = RunState::load(&runner.state.id).unwrap();
13488        assert!(disk.driver_exited);
13489        assert_eq!(disk.driver_pid, None);
13490        assert!(disk.status.resumable());
13491        assert!(runner.state.driver_exited);
13492        assert_eq!(runner.state.driver_pid, None);
13493        runner.state.save().unwrap();
13494        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13495    }
13496
13497    #[tokio::test]
13498    #[cfg(unix)]
13499    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13500        let tmp = tempfile::tempdir().unwrap();
13501        let mut runner = registered_runner(tmp.path()).await;
13502        let winner = runner.state.candidates[0].clone();
13503        let cache = tmp.path().join("cache");
13504        let started = tmp.path().join("pre-gate-started");
13505        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13506        runner.state.config.verify.gate =
13507            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13508        let home = crate::run::home();
13509        let owner =
13510            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13511        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13512            crate::cache::AcquireOutcome::Acquired(g) => g,
13513            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13514        };
13515        let mut gate = Box::pin(runner.run_gate(&winner));
13516        assert!(
13517            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13518                .await
13519                .is_err()
13520        );
13521        assert!(!started.exists(), "pre-gate must wait for the cache too");
13522        // Simulate the wave that already owns the cache entering invoke.
13523        // It must obtain the editor reservation and finish without waiting
13524        // for the gate that is queued behind its cache.
13525        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13526        let editor = crate::repo_lock::SeatLease::checkout_with(
13527            &winner.worktree,
13528            true,
13529            crate::config::test_coordination().as_ref(),
13530            &wait,
13531        )
13532        .await
13533        .unwrap();
13534        drop(editor);
13535        guard.release();
13536        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13537            .await
13538            .unwrap()
13539            .unwrap();
13540        drop(gate);
13541        assert!(outcomes.iter().all(CommandOutcome::ok));
13542        assert!(started.exists());
13543    }
13544
13545    #[tokio::test]
13546    #[cfg(unix)]
13547    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13548        use std::os::fd::AsRawFd as _;
13549
13550        ask_test_home();
13551        let tmp = tempfile::tempdir().unwrap();
13552        let repo = tmp.path().join("repo");
13553        std::fs::create_dir_all(&repo).unwrap();
13554        init_repo(&repo);
13555        let common = repo.join(".git");
13556        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13557        let mut runner = runner_at(RunStatus::Prep);
13558        runner.state.repo = repo.clone();
13559        runner.state.config.coordination = crate::config::test_coordination();
13560        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13561        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13562        runner.state.config.graph.advise = false;
13563        runner.roles.implementers = vec![spec("implement")];
13564        runner.roles.judges = vec![spec("judge")];
13565        runner.prep().await.unwrap();
13566        let winner = runner.state.candidates[0].clone();
13567        // Emulate edits already left by a completed seat. The fixture CLI
13568        // only reports completion; it performs no repository operations.
13569        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13570        let completed = tmp.path().join("seat-completed");
13571        runner.roles.implementers[0].command = vec![
13572            "sh".into(),
13573            "-c".into(),
13574            format!("touch '{}'; echo completed", completed.display()),
13575        ];
13576        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13577        assert_eq!(
13578            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13579            0
13580        );
13581        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13582        let mut implement = Box::pin(runner.implement());
13583        tokio::select! {
13584            result = &mut implement => panic!("rescue must wait: {result:?}"),
13585            result = tokio::time::timeout(Duration::from_secs(10), async {
13586                while !completed.exists() {
13587                    tokio::time::sleep(Duration::from_millis(20)).await;
13588                }
13589            }) => result.unwrap(),
13590        }
13591        assert!(
13592            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13593                .await
13594                .is_err()
13595        );
13596        assert_eq!(
13597            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13598            before
13599        );
13600        assert_eq!(
13601            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13602            0
13603        );
13604        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13605            .await
13606            .unwrap()
13607            .unwrap();
13608        drop(implement);
13609        assert_ne!(
13610            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13611            before
13612        );
13613        assert!(git::is_clean(&winner.worktree).await.unwrap());
13614
13615        let started = tmp.path().join("pre-gate-started");
13616        let proceed = tmp.path().join("pre-gate-proceed");
13617        let checked = tmp.path().join("gate-ran");
13618        runner.state.config.verify.pre_gate = vec![format!(
13619            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13620            started.display(),
13621            proceed.display()
13622        )];
13623        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13624        assert_eq!(
13625            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13626            0
13627        );
13628        let mut gate = Box::pin(runner.run_gate(&winner));
13629        assert!(
13630            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13631                .await
13632                .is_err()
13633        );
13634        assert!(!started.exists());
13635        assert!(!checked.exists());
13636        assert!(git::is_clean(&winner.worktree).await.unwrap());
13637        assert_eq!(
13638            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13639            0
13640        );
13641        tokio::select! {
13642            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13643            result = tokio::time::timeout(Duration::from_secs(10), async {
13644                while !started.exists() {
13645                    tokio::time::sleep(Duration::from_millis(20)).await;
13646                }
13647            }) => result.unwrap(),
13648        }
13649        assert_eq!(
13650            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13651            -1,
13652            "mechanical edits must hold the real lock"
13653        );
13654        std::fs::write(proceed, "ready").unwrap();
13655        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13656            .await
13657            .unwrap()
13658            .unwrap();
13659        drop(gate);
13660        assert!(outcomes.iter().all(CommandOutcome::ok));
13661        assert!(checked.exists());
13662        assert!(runner.state.pre_gate_commit.is_some());
13663        assert!(git::is_clean(&winner.worktree).await.unwrap());
13664        assert_eq!(
13665            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13666            0,
13667            "gate completion must release coordination"
13668        );
13669        drop(outsider);
13670        std::fs::remove_file(common.join("updater.lock")).unwrap();
13671        std::fs::create_dir(common.join("updater.lock")).unwrap();
13672        std::fs::remove_file(&started).unwrap();
13673        std::fs::remove_file(&checked).unwrap();
13674        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13675        let error = runner.run_gate(&winner).await.unwrap_err();
13676        assert!(
13677            error
13678                .to_string()
13679                .contains("acquire mandatory repository lock")
13680        );
13681        assert!(
13682            !started.exists(),
13683            "failed acquisition must prevent pre-gate"
13684        );
13685        assert!(!checked.exists(), "failed acquisition must prevent gate");
13686        assert!(git::is_clean(&winner.worktree).await.unwrap());
13687        assert_eq!(
13688            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13689            head
13690        );
13691    }
13692
13693    #[tokio::test]
13694    #[cfg(unix)]
13695    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13696        use std::os::fd::AsRawFd as _;
13697        use std::time::Duration;
13698
13699        ask_test_home();
13700        let tmp = tempfile::tempdir().unwrap();
13701        let repo = tmp.path().join("repo");
13702        std::fs::create_dir_all(&repo).unwrap();
13703        init_repo(&repo);
13704        let common = repo.join(".git");
13705        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13706        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13707        assert_eq!(
13708            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13709            0
13710        );
13711        let mut runner = runner_at(RunStatus::Prep);
13712        runner.state.repo = repo.clone();
13713        runner.state.config.coordination = crate::config::test_coordination();
13714        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13715        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13716        runner.state.config.graph.advise = true;
13717        runner.state.config.graph.advisors = 1;
13718        runner.roles.implementers = vec![spec("implement")];
13719        runner.roles.judges = vec![spec("judge")];
13720        let root = runner.state.worktree_root();
13721        let started = tmp.path().join("setup-started");
13722        let proceed = tmp.path().join("setup-proceed");
13723        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13724            run: Some(format!(
13725                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13726                started.display(),
13727                proceed.display()
13728            )),
13729            ..Default::default()
13730        }];
13731        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13732        let config = git::git(&repo, &["config", "--local", "--list"])
13733            .await
13734            .unwrap();
13735        let mut prep = Box::pin(runner.prep());
13736        assert!(
13737            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13738                .await
13739                .is_err()
13740        );
13741        assert!(
13742            !root.exists(),
13743            "contention must prevent even checkout creation"
13744        );
13745        assert!(!started.exists(), "contention must prevent setup");
13746        assert_eq!(
13747            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13748            branches
13749        );
13750        assert_eq!(
13751            git::git(&repo, &["config", "--local", "--list"])
13752                .await
13753                .unwrap(),
13754            config
13755        );
13756        drop(lock);
13757        tokio::select! {
13758            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13759            result = tokio::time::timeout(Duration::from_secs(10), async {
13760                while !started.exists() {
13761                    tokio::time::sleep(Duration::from_millis(20)).await;
13762                }
13763            }) => result.unwrap(),
13764        }
13765        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13766        assert_eq!(
13767            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13768            -1
13769        );
13770        assert!(
13771            tokio::time::timeout(
13772                Duration::from_millis(100),
13773                crate::repo_lock::SeatLease::for_checkout(
13774                    &repo,
13775                    true,
13776                    crate::config::test_coordination().as_ref()
13777                )
13778            )
13779            .await
13780            .is_err(),
13781            "preparation must also exclude local editors"
13782        );
13783        std::fs::write(proceed, "ready").unwrap();
13784        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13785            .await
13786            .unwrap()
13787            .unwrap();
13788        drop(prep);
13789        assert_eq!(runner.state.prep_complete, Some(true));
13790        assert!(root.join("judge-1").exists());
13791        assert!(root.join("advisor-1").exists());
13792        assert_eq!(
13793            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13794            0
13795        );
13796    }
13797
13798    #[tokio::test]
13799    #[cfg(unix)]
13800    async fn preparation_lock_failure_prevents_repository_changes() {
13801        ask_test_home();
13802        let tmp = tempfile::tempdir().unwrap();
13803        let repo = tmp.path().join("repo");
13804        std::fs::create_dir_all(&repo).unwrap();
13805        init_repo(&repo);
13806        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13807        // A directory at the required lock path makes opening the real lock
13808        // fail, rather than merely exercising an invalid registration.
13809        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13810        let mut runner = runner_at(RunStatus::Prep);
13811        runner.state.repo = repo.clone();
13812        runner.state.config.coordination = crate::config::test_coordination();
13813        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13814        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13815        runner.roles.implementers = vec![spec("implement")];
13816        let error = runner.prep().await.unwrap_err();
13817        assert!(
13818            error
13819                .to_string()
13820                .contains("acquire mandatory repository lock")
13821        );
13822        assert!(!runner.state.worktree_root().exists());
13823        assert!(runner.state.candidates.is_empty());
13824        assert!(!repo.join(".git/config.worktree").exists());
13825        assert_eq!(
13826            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13827            "* main"
13828        );
13829    }
13830
13831    #[tokio::test]
13832    async fn preparation_records_creation_intent_before_git_can_fail() {
13833        ask_test_home();
13834        let tmp = tempfile::tempdir().unwrap();
13835        let repo = tmp.path().join("repo");
13836        std::fs::create_dir_all(&repo).unwrap();
13837        init_repo(&repo);
13838        let mut runner = runner_at(RunStatus::Prep);
13839        runner.state.repo = repo.clone();
13840        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13841        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13842        runner.state.config.blind.commit_msg_hook = false;
13843        runner.roles.implementers = vec![spec("one")];
13844        let label = blind::assign_labels(1, runner.state.seed)[0];
13845        let branch = runner.state.branch_for(label);
13846        // A conflicting branch makes Git refuse creation before a candidate
13847        // can be registered. The durable intent must still identify it.
13848        git::git(&repo, &["branch", &branch]).await.unwrap();
13849        assert!(runner.prep().await.is_err());
13850        let saved = RunState::load(&runner.state.id).unwrap();
13851        assert!(saved.candidates.is_empty());
13852        let event = saved
13853            .events
13854            .iter()
13855            .find(|e| e.node == "prep-intent")
13856            .unwrap();
13857        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13858        assert_eq!(intent["branch"], branch);
13859        assert_eq!(intent["start"], runner.state.base_commit);
13860        assert_eq!(
13861            intent["worktree"],
13862            runner
13863                .state
13864                .worktree_root()
13865                .join(format!("cand-{label}"))
13866                .to_string_lossy()
13867                .as_ref()
13868        );
13869        assert_eq!(intent["picks"], serde_json::json!([]));
13870    }
13871
13872    #[tokio::test]
13873    async fn invalid_pinned_sources_create_no_partial_worktrees() {
13874        ask_test_home();
13875        let tmp = tempfile::tempdir().unwrap();
13876        let repo = tmp.path().join("repo");
13877        std::fs::create_dir_all(&repo).unwrap();
13878        init_repo(&repo);
13879        let mut runner = runner_at(RunStatus::Prep);
13880        runner.state.repo = repo.clone();
13881        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13882        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13883        runner.state.sources.pick_commits.push(refs::Input {
13884            token: "missing".into(),
13885            sha: "0".repeat(40),
13886            provenance: refs::Provenance::ExplicitCli,
13887        });
13888        assert!(runner.prep().await.is_err());
13889        assert!(runner.state.candidates.is_empty());
13890        assert!(!runner.state.worktree_root().exists());
13891    }
13892
13893    #[tokio::test]
13894    async fn prose_and_legacy_inference_never_import_into_candidates() {
13895        ask_test_home();
13896        let tmp = tempfile::tempdir().unwrap();
13897        let repo = tmp.path().join("repo");
13898        std::fs::create_dir_all(&repo).unwrap();
13899        init_repo(&repo);
13900        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13901        git::git(&repo, &["checkout", "-b", "magi/history/A"])
13902            .await
13903            .unwrap();
13904        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13905        git::git(&repo, &["add", "."]).await.unwrap();
13906        git::git(&repo, &["commit", "-m", "historical implementation"])
13907            .await
13908            .unwrap();
13909        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13910        git::git(&repo, &["checkout", "main"]).await.unwrap();
13911        for instruction in [
13912            format!("Example: magi/history/A at {source}"),
13913            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13914            format!("Never import magi/history/A or {source}"),
13915            format!("Conversation answer: I mentioned {source} only as background"),
13916        ] {
13917            let mut runner = runner_at(RunStatus::Prep);
13918            runner.state.repo = repo.clone();
13919            runner.state.base_commit = base.clone();
13920            runner.state.instruction = instruction.clone();
13921            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13922            runner.state.config.blind.commit_msg_hook = false;
13923            runner.state.seeds = vec![refs::Seed {
13924                token: "magi/history/A".into(),
13925                sha: source.clone(),
13926                kind: refs::SeedKind::Unmerged,
13927                branch: true,
13928                detail: String::new(),
13929            }];
13930            runner.roles.implementers = vec![spec("one")];
13931            assert_eq!(seeded_instruction(&runner.state), instruction);
13932            // Model an old record with inferred seeds but no structured input field.
13933            let mut legacy = serde_json::to_value(&runner.state).unwrap();
13934            legacy.as_object_mut().unwrap().remove("sources");
13935            runner.state = serde_json::from_value(legacy).unwrap();
13936            assert_eq!(runner.state.sources, refs::Sources::default());
13937            runner.state.save().unwrap();
13938            runner.state = RunState::load(&runner.state.id).unwrap();
13939            runner.prep().await.unwrap();
13940            for candidate in &runner.state.candidates {
13941                assert_eq!(
13942                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13943                    base
13944                );
13945                assert!(!candidate.worktree.join("unwanted").exists());
13946            }
13947        }
13948    }
13949
13950    #[tokio::test]
13951    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13952        ask_test_home();
13953        let tmp = tempfile::tempdir().unwrap();
13954        let repo = tmp.path().join("repo");
13955        std::fs::create_dir_all(&repo).unwrap();
13956        init_repo(&repo);
13957        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13958        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13959        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13960        git::git(&repo, &["add", "."]).await.unwrap();
13961        git::git(&repo, &["commit", "-m", "required seed"])
13962            .await
13963            .unwrap();
13964        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13965        git::git(&repo, &["checkout", "main"]).await.unwrap();
13966        let mut runner = runner_at(RunStatus::Prep);
13967        runner.state.repo = repo.clone();
13968        runner.state.base_commit = base.clone();
13969        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13970        runner.state.config.graph.advise = false;
13971        runner.state.config.blind.commit_msg_hook = false;
13972        runner.state.sources =
13973            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13974                .await
13975                .unwrap();
13976        runner.roles.implementers = vec![spec("one"), spec("two")];
13977        runner.roles.judges = vec![spec("judge")];
13978        let label = blind::assign_labels(2, runner.state.seed)[0];
13979        let branch = runner.state.branch_for(label);
13980        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13981        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13982            .await
13983            .unwrap();
13984        runner.state.candidates.push(Candidate {
13985            index: 0,
13986            label,
13987            agent: "one".into(),
13988            branch,
13989            worktree,
13990            summary: String::new(),
13991            stat: String::new(),
13992            files: 0,
13993            commits: 0,
13994            empty: false,
13995            failed: None,
13996            verified_noop: None,
13997            duration_ms: 0,
13998            folded: false,
13999        });
14000        runner.state.save().unwrap();
14001        // Reload the checkpoint immediately before its first cherry-pick.
14002        runner.state = RunState::load(&runner.state.id).unwrap();
14003        assert_eq!(runner.state.prep_complete, Some(false));
14004        runner.prep().await.unwrap();
14005        assert_eq!(runner.state.prep_complete, Some(true));
14006        assert_eq!(runner.state.candidates.len(), 2);
14007        for candidate in &runner.state.candidates {
14008            assert_eq!(
14009                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14010                git::tree_of(&repo, &seed).await.unwrap()
14011            );
14012            assert_eq!(
14013                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14014                "required commit"
14015            );
14016        }
14017        assert!(runner.state.worktree_root().join("judge-1").exists());
14018        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14019        runner.prep().await.unwrap();
14020        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14021
14022        runner.state.prep_complete = Some(false);
14023        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14024            run: Some(format!(
14025                "test -f '{}'",
14026                tmp.path().join("dependency-ready").display()
14027            )),
14028            ..Default::default()
14029        }];
14030        assert!(runner.prep().await.is_err());
14031        assert!(runner.state.prep_failure.is_some());
14032        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14033        runner.state = RunState::load(&runner.state.id).unwrap();
14034        assert!(
14035            runner.prep().await.is_err(),
14036            "correcting the cause alone must not auto-retry"
14037        );
14038        let failed_at = runner.state.events.last().unwrap().at;
14039        assert!(
14040            runner.retry_preparation(failed_at).is_err(),
14041            "old approvals cannot unlock the hold"
14042        );
14043        let mut task = crate::queue::Task::new(
14044            "retry".into(),
14045            "retry setup".into(),
14046            repo.clone(),
14047            crate::queue::Source::Human,
14048        );
14049        task.start(runner.state.id.clone());
14050        task.hold_machine(Some("setup failed".into()));
14051        task.release_operator("cli", None);
14052        runner
14053            .retry_preparation(task.resumption.as_ref().unwrap().at)
14054            .unwrap();
14055        runner.prep().await.unwrap();
14056        assert!(runner.state.prep_failure.is_none());
14057        assert_eq!(runner.state.prep_complete, Some(true));
14058        assert_eq!(runner.state.candidates.len(), 2);
14059        assert!(runner.state.worktree_root().join("judge-1").exists());
14060        for candidate in &runner.state.candidates {
14061            assert_eq!(
14062                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14063                git::tree_of(&repo, &seed).await.unwrap()
14064            );
14065        }
14066    }
14067
14068    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14069        let mut heads = Vec::new();
14070        for candidate in candidates {
14071            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14072        }
14073        heads
14074    }
14075
14076    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14077        ask_test_home();
14078        let tmp = tempfile::tempdir().unwrap();
14079        let repo = tmp.path().join("repo");
14080        std::fs::create_dir(&repo).unwrap();
14081        init_repo(&repo);
14082        let mut runner = runner_at(RunStatus::Reviewing);
14083        runner.state.repo = repo.clone();
14084        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14085        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14086        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14087            let branch = format!("candidate-{label}");
14088            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14089            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14090                .await
14091                .unwrap();
14092            if label == 'B' {
14093                std::fs::write(worktree.join("change"), "winner").unwrap();
14094                git::rescue_commit(&worktree, "candidate change")
14095                    .await
14096                    .unwrap();
14097            }
14098            runner.state.candidates.push(Candidate {
14099                index,
14100                label,
14101                agent: "alpha".into(),
14102                branch,
14103                worktree,
14104                summary: "completed reply".into(),
14105                stat: String::new(),
14106                files: usize::from(label == 'B'),
14107                commits: usize::from(label == 'B'),
14108                empty: label == 'A',
14109                failed: None,
14110                verified_noop: None,
14111                duration_ms: 1,
14112                folded: false,
14113            });
14114        }
14115        runner.state.judge_skipped = true;
14116        runner.state.tally = Some(Tally {
14117            first_choice: BTreeMap::new(),
14118            borda: BTreeMap::new(),
14119            winner: 'B',
14120            rankings: 0,
14121            unanimous_initial: false,
14122            deliberated: false,
14123            changed_votes: 0,
14124            unanimous_final: false,
14125            tie_break: None,
14126            judges: 0,
14127            present: 0,
14128            quorum: 0,
14129            met_quorum: true,
14130            uncontested: Some("only B produced a change".into()),
14131        });
14132        runner.state.save().unwrap();
14133        (tmp, runner)
14134    }
14135
14136    #[tokio::test]
14137    #[cfg(unix)]
14138    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14139        for (solo, failed_checkpoint) in
14140            [(true, false), (false, false), (true, true), (false, true)]
14141        {
14142            let (_tmp, mut runner) = preservation_resume_fixture().await;
14143            if solo {
14144                runner.state.candidates.retain(|c| c.label == 'A');
14145            }
14146            runner.state.candidates[0].empty = false;
14147            runner.state.candidates[0].summary.clear();
14148            runner.state.tally = None;
14149            runner.state.judge_skipped = false;
14150            runner.state.status = RunStatus::Implementing;
14151            runner.state.config.disk.min_free_bytes = u64::MAX;
14152            let mut implementer = spec("alpha");
14153            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14154            runner.roles.implementers = vec![implementer];
14155            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14156            if failed_checkpoint {
14157                std::fs::create_dir_all(&patch).unwrap();
14158            }
14159            runner.state.save().unwrap();
14160            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14161            runner.state = RunState::load(&runner.state.id).unwrap();
14162            assert_eq!(runner.state.status, RunStatus::Blocked);
14163            assert!(runner.state.interrupted_implementations.contains(&'A'));
14164            assert_eq!(runner.state.candidates[0].commits, 0);
14165            assert!(!runner.state.candidates[0].empty);
14166            assert!(runner.state.candidates[0].failed.is_none());
14167            if failed_checkpoint {
14168                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14169                std::fs::remove_dir(patch).unwrap();
14170            }
14171            runner.state.resource_block = None;
14172            runner.state.config.disk.min_free_bytes = 0;
14173            runner.roles.implementers[0].command = vec![
14174                "sh".into(),
14175                "-c".into(),
14176                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14177            ];
14178            runner.implement().await.unwrap();
14179            assert!(runner.state.interrupted_implementations.is_empty());
14180            assert!(runner.state.candidate_preservation.is_empty());
14181            assert_eq!(runner.state.candidates[0].commits, 1);
14182            assert!(!runner.state.candidates[0].empty);
14183            assert!(
14184                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14185                    .unwrap()
14186                    .contains("retried empty interrupted seat")
14187            );
14188            if !solo {
14189                assert_eq!(runner.state.candidates[1].commits, 1);
14190            }
14191        }
14192    }
14193
14194    #[tokio::test]
14195    #[cfg(unix)]
14196    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14197        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14198            let (_tmp, mut runner) = preservation_resume_fixture().await;
14199            runner.state.candidates.retain(|c| c.label == 'A');
14200            runner.state.candidates[0].empty = false;
14201            runner.state.candidates[0].summary.clear();
14202            runner.state.tally = None;
14203            runner.state.judge_skipped = false;
14204            runner.state.status = RunStatus::Implementing;
14205            // Force the real post-spawn watchdog, independently of available
14206            // disk space. Dirty/committed work predates its first measurement.
14207            runner.state.config.disk.min_free_bytes = u64::MAX;
14208            let wt = runner.state.candidates[0].worktree.clone();
14209            std::fs::write(
14210                wt.join("interrupted-source"),
14211                "retain implementation work\n",
14212            )
14213            .unwrap();
14214            if committed {
14215                git::rescue_commit(&wt, "implementation checkpoint")
14216                    .await
14217                    .unwrap();
14218            }
14219            let lock = git::git(
14220                &wt,
14221                &[
14222                    "rev-parse",
14223                    "--path-format=absolute",
14224                    "--git-path",
14225                    "index.lock",
14226                ],
14227            )
14228            .await
14229            .unwrap();
14230            if block_rescue {
14231                std::fs::write(&lock, "fixture-owned lock").unwrap();
14232            }
14233            let mut implementer = spec("alpha");
14234            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14235            runner.roles.implementers = vec![implementer];
14236            runner.state.save().unwrap();
14237            let result = runner.implement().await;
14238            assert_eq!(result.is_err(), block_rescue);
14239            assert_eq!(
14240                runner.state.resource_block.as_ref().unwrap().kind,
14241                "disk pressure"
14242            );
14243            assert!(
14244                runner
14245                    .state
14246                    .events
14247                    .iter()
14248                    .any(|e| e.message.contains("agent execution interrupted"))
14249            );
14250            assert!(
14251                runner
14252                    .state
14253                    .dir()
14254                    .join("artifacts/candidate-A-recovery/staged.patch")
14255                    .exists()
14256            );
14257            if block_rescue {
14258                let saved = RunState::load(&runner.state.id).unwrap();
14259                assert!(saved.candidate_preservation.contains_key(&'A'));
14260                assert!(!saved.candidates[0].empty);
14261                assert_eq!(
14262                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14263                    "retain implementation work\n"
14264                );
14265                std::fs::remove_file(lock).unwrap();
14266                runner.state = saved;
14267            } else {
14268                assert_eq!(runner.state.status, RunStatus::Blocked);
14269                assert_eq!(runner.state.candidates[0].commits, 1);
14270            }
14271            // Recovery must save the candidate before deciding whether another
14272            // implementation is needed. No implementer is available on retry.
14273            runner.roles.implementers.clear();
14274            runner.state.config.disk.min_free_bytes = 0;
14275            runner.state.resource_block = None;
14276            runner.implement().await.unwrap();
14277            assert!(runner.state.candidate_preservation.is_empty());
14278            assert_eq!(runner.state.candidates[0].commits, 1);
14279            assert!(!runner.state.candidates[0].empty);
14280            assert!(git::is_clean(&wt).await.unwrap());
14281            assert!(
14282                crate::run::read_artifact(&runner.state, "cand-A.patch")
14283                    .unwrap()
14284                    .contains("retain implementation work")
14285            );
14286        }
14287    }
14288
14289    #[tokio::test]
14290    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14291        let (_tmp, mut runner) = preservation_resume_fixture().await;
14292        // Schema 15 already confirmed the unfolded no-change result.
14293        runner.implement().await.unwrap();
14294        assert!(runner.state.candidate_preservation.is_empty());
14295        assert!(
14296            !runner
14297                .state
14298                .dir()
14299                .join("artifacts/candidate-A-recovery")
14300                .exists()
14301        );
14302        runner.fold_losers().await.unwrap();
14303        assert!(!runner.state.candidates[0].worktree.exists());
14304        for schema in [14, crate::run::SCHEMA] {
14305            runner.state.schema = schema;
14306            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14307            runner.state.save().unwrap();
14308            runner.state = RunState::load(&runner.state.id).unwrap();
14309            runner.implement().await.unwrap();
14310            assert!(runner.state.candidate_preservation.is_empty());
14311            assert!(runner.state.candidates[0].folded);
14312            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14313        }
14314    }
14315
14316    #[tokio::test]
14317    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14318        let (_tmp, mut runner) = preservation_resume_fixture().await;
14319        runner.state.candidates.retain(|c| c.label == 'A');
14320        runner.state.tally = None;
14321        runner.state.judge_skipped = false;
14322        let candidate = &mut runner.state.candidates[0];
14323        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14324        candidate.failed = Some("deployment validation unavailable".into());
14325        std::fs::write(
14326            candidate.worktree.join("repair"),
14327            "future launcher policy\n",
14328        )
14329        .unwrap();
14330        runner
14331            .state
14332            .candidate_preservation
14333            .insert('A', "awaiting persistence".into());
14334        runner.persist_candidates().await.unwrap();
14335        let candidate = &runner.state.candidates[0];
14336        assert!(!candidate.empty);
14337        assert_eq!(candidate.commits, 1);
14338        assert_eq!(candidate.files, 1);
14339        assert!(candidate.failed.is_none());
14340        assert!(candidate.summary.contains("EPERM"));
14341        assert!(
14342            crate::run::read_artifact(&runner.state, "cand-A.patch")
14343                .unwrap()
14344                .contains("future launcher policy")
14345        );
14346        assert!(runner.state.candidate_preservation.is_empty());
14347    }
14348
14349    #[tokio::test]
14350    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14351        let (_tmp, mut runner) = preservation_resume_fixture().await;
14352        let wt = runner.state.candidates[0].worktree.clone();
14353        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14354        runner.state.schema = 14;
14355        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14356        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14357        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14358        for _ in 0..2 {
14359            assert!(runner.implement().await.is_err());
14360            assert_eq!(runner.state.status, RunStatus::Blocked);
14361            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14362            assert!(
14363                runner.state.candidates[0].empty,
14364                "old verdict must remain unusable"
14365            );
14366            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14367            assert!(runner.state.judge_skipped);
14368            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14369            assert!(
14370                crate::run::read_artifact(&runner.state, "cand-A.patch")
14371                    .unwrap()
14372                    .contains("owned edits")
14373            );
14374            assert!(wt.exists());
14375            assert_eq!(
14376                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14377                    .await
14378                    .unwrap(),
14379                1
14380            );
14381            runner.state = RunState::load(&runner.state.id).unwrap();
14382        }
14383    }
14384
14385    #[tokio::test]
14386    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14387        let (_tmp, mut runner) = preservation_resume_fixture().await;
14388        let wt = runner.state.candidates[0].worktree.clone();
14389        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14390        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14391        let index_lock = PathBuf::from(
14392            git::git(
14393                &wt,
14394                &[
14395                    "rev-parse",
14396                    "--path-format=absolute",
14397                    "--git-path",
14398                    "index.lock",
14399                ],
14400            )
14401            .await
14402            .unwrap(),
14403        );
14404        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14405        runner.state.schema = 14;
14406        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14407        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14408        assert!(runner.implement().await.is_err());
14409        assert!(runner.after_implement().is_err());
14410        assert!(runner.fold_losers().await.is_err());
14411        assert_eq!(
14412            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14413            "failed seat edits"
14414        );
14415        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14416        runner.state = RunState::load(&runner.state.id).unwrap();
14417        std::fs::remove_file(index_lock).unwrap();
14418        // Recovery finds changes that the old uncontested result never judged.
14419        assert!(runner.implement().await.is_err());
14420        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14421        assert!(runner.state.candidates[0].empty);
14422        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14423        assert!(
14424            crate::run::read_artifact(&runner.state, "cand-A.patch")
14425                .unwrap()
14426                .contains("failed seat edits")
14427        );
14428        assert!(wt.exists());
14429    }
14430
14431    #[tokio::test]
14432    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14433        let (_tmp, mut runner) = preservation_resume_fixture().await;
14434        let repo = runner.state.repo.clone();
14435        let common = git::git(
14436            &repo,
14437            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14438        )
14439        .await
14440        .unwrap();
14441        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14442        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14443        assert!(runner.fold_losers().await.is_err());
14444        assert!(!runner.state.candidates[0].worktree.exists());
14445        assert!(!runner.state.candidates[0].folded);
14446        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14447        runner.state = RunState::load(&runner.state.id).unwrap();
14448        std::fs::remove_file(branch_lock).unwrap();
14449        let judge = runner.state.worktree_root().join("judge-1");
14450        git::worktree_add_detached(&repo, &judge, "main")
14451            .await
14452            .unwrap();
14453        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14454        runner.roles.judges = vec![spec("judge")];
14455        assert!(runner.fold_losers().await.is_err());
14456        runner.state = RunState::load(&runner.state.id).unwrap();
14457        assert!(
14458            runner.state.candidates[0].folded,
14459            "candidate progress saved before judge failure"
14460        );
14461        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14462        assert!(judge.join("uncommitted").exists());
14463        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14464        runner.fold_losers().await.unwrap();
14465        assert!(!judge.exists());
14466        runner.fold_losers().await.unwrap();
14467    }
14468
14469    #[tokio::test]
14470    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14471        let (_tmp, mut runner) = preservation_resume_fixture().await;
14472        runner.state.candidates.retain(|c| c.label == 'B');
14473        runner.state.tally = None;
14474        runner.state.judge_skipped = false;
14475        runner.state.prep_complete = Some(true);
14476        runner.state.config.graph.advise = false;
14477        runner.state.config.graph.selected_implementer = Some("alpha".into());
14478        let wt = runner.state.candidates[0].worktree.clone();
14479        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14480        runner
14481            .state
14482            .candidate_preservation
14483            .insert('B', "awaiting persistence".into());
14484        let index_lock = git::git(
14485            &wt,
14486            &[
14487                "rev-parse",
14488                "--path-format=absolute",
14489                "--git-path",
14490                "index.lock",
14491            ],
14492        )
14493        .await
14494        .unwrap();
14495        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14496        runner.state.save().unwrap();
14497        assert!(runner.execute_graph().await.is_err());
14498        assert_eq!(runner.state.status, RunStatus::Blocked);
14499        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14500        std::fs::remove_file(index_lock).unwrap();
14501        // Park at the next graph boundary: this tests the real selected-seat
14502        // hold guard without dispatching reviewers or publishing anything.
14503        for _ in 0..2 {
14504            runner.state = RunState::load(&runner.state.id).unwrap();
14505            runner.pause.park();
14506            runner.execute_graph().await.unwrap();
14507            assert_eq!(runner.state.status, RunStatus::Implementing);
14508            assert!(runner.state.candidate_preservation.is_empty());
14509            assert_eq!(runner.state.candidates[0].commits, 2);
14510            assert!(runner.state.tally.is_none());
14511        }
14512        runner.implement().await.unwrap();
14513        assert_eq!(runner.state.status, RunStatus::Judging);
14514        // A genuine selected-implementer hold must still require release.
14515        runner.state.status = RunStatus::Blocked;
14516        runner.state.candidates[0].failed = Some("implementation failed".into());
14517        runner.state.save().unwrap();
14518        let error = runner.execute_graph().await.unwrap_err();
14519        assert!(
14520            error
14521                .to_string()
14522                .contains("selected implementation is held")
14523        );
14524    }
14525
14526    #[tokio::test]
14527    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14528        let (_tmp, mut runner) = preservation_resume_fixture().await;
14529        let repo = runner.state.repo.clone();
14530        let candidate = runner.state.candidates[0].clone();
14531        // Remove only the disposable fixture directory, leaving Git's linked
14532        // worktree registration and branch intact, as in a vanished checkout.
14533        let registered = git::worktree_holding(&repo, &candidate.branch)
14534            .await
14535            .unwrap()
14536            .unwrap();
14537        assert_eq!(
14538            registered.canonicalize().unwrap(),
14539            candidate.worktree.canonicalize().unwrap()
14540        );
14541        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14542        assert_eq!(
14543            git::worktree_holding(&repo, &candidate.branch)
14544                .await
14545                .unwrap(),
14546            Some(registered)
14547        );
14548        assert!(
14549            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14550            "the stale registration initially keeps the branch checked out"
14551        );
14552        runner.fold_losers().await.unwrap();
14553        assert!(runner.state.candidates[0].folded);
14554        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14555        assert_eq!(
14556            git::worktree_holding(&repo, &candidate.branch)
14557                .await
14558                .unwrap(),
14559            None
14560        );
14561        runner.state = RunState::load(&runner.state.id).unwrap();
14562        runner.fold_losers().await.unwrap();
14563    }
14564
14565    #[tokio::test]
14566    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14567        let (_tmp, mut runner) = preservation_resume_fixture().await;
14568        let repo = runner.state.repo.clone();
14569        let candidate = runner.state.candidates[0].clone();
14570        let winner = runner.state.candidates[1].clone();
14571        let registered = git::worktree_holding(&repo, &candidate.branch)
14572            .await
14573            .unwrap()
14574            .unwrap();
14575        assert_eq!(
14576            registered.canonicalize().unwrap(),
14577            candidate.worktree.canonicalize().unwrap()
14578        );
14579        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14580        assert_eq!(
14581            git::worktree_holding(&repo, &candidate.branch)
14582                .await
14583                .unwrap(),
14584            Some(registered)
14585        );
14586        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14587
14588        for _ in 0..2 {
14589            fold_run(&mut runner.state, false, &crate::run::home())
14590                .await
14591                .unwrap();
14592            runner.state = RunState::load(&runner.state.id).unwrap();
14593            assert!(runner.state.candidates[0].folded);
14594            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14595            assert_eq!(
14596                git::worktree_holding(&repo, &candidate.branch)
14597                    .await
14598                    .unwrap(),
14599                None
14600            );
14601            assert!(!runner.state.candidates[1].folded);
14602            assert!(winner.worktree.exists());
14603            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14604        }
14605    }
14606
14607    #[tokio::test]
14608    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14609        ask_test_home();
14610        let tmp = tempfile::tempdir().unwrap();
14611        let repo = tmp.path().join("repo");
14612        std::fs::create_dir(&repo).unwrap();
14613        init_repo(&repo);
14614        let mut runner = runner_at(RunStatus::Implementing);
14615        runner.state.repo = repo.clone();
14616        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14617        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14618        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14619            let branch = format!("candidate-{label}");
14620            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14621            git::worktree_add_branch(&repo, &wt, &branch, "main")
14622                .await
14623                .unwrap();
14624            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14625            git::git(&wt, &["add", "README.md"]).await.unwrap();
14626            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14627            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14628            runner.state.candidates.push(Candidate {
14629                index,
14630                label,
14631                agent: "alpha".to_owned(),
14632                branch,
14633                worktree: wt,
14634                summary: "completed implementation".to_owned(),
14635                stat: String::new(),
14636                files: 0,
14637                commits: 0,
14638                empty: false,
14639                failed: None,
14640                verified_noop: None,
14641                duration_ms: 42,
14642                folded: false,
14643            });
14644            runner
14645                .state
14646                .candidate_preservation
14647                .insert(label, "awaiting persistence".to_owned());
14648        }
14649        let wt = runner.state.candidates[0].worktree.clone();
14650        let index_lock = git::git(
14651            &wt,
14652            &[
14653                "rev-parse",
14654                "--path-format=absolute",
14655                "--git-path",
14656                "index.lock",
14657            ],
14658        )
14659        .await
14660        .unwrap();
14661        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14662        runner.state.save().unwrap();
14663        assert!(runner.persist_candidates().await.is_err());
14664        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14665        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14666        assert_eq!(
14667            runner.state.candidates[1].commits, 1,
14668            "other results are still collected"
14669        );
14670        assert!(
14671            !runner.state.candidates[0].empty,
14672            "failed rescue is not no change"
14673        );
14674        assert_eq!(
14675            runner.state.candidates[0].summary,
14676            "completed implementation"
14677        );
14678        assert!(runner.after_implement().is_err());
14679        let home = tmp.path().join("fold-home");
14680        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14681        assert!(wt.join("new-file").exists());
14682        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14683        assert!(recovery.join("staged.patch").exists());
14684        assert!(recovery.join("unstaged.patch").exists());
14685        assert_eq!(
14686            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14687            "untracked edits\n"
14688        );
14689        runner.state.begin_pass();
14690        runner.state.end_pass_parked(ParkCause::Upgrade {
14691            from: "1.0".into(),
14692            to: Some("1.1".into()),
14693        });
14694        runner.state.save().unwrap();
14695        runner.state = RunState::load(&runner.state.id).unwrap();
14696        assert!(matches!(
14697            runner.state.passes.last().unwrap().end,
14698            Some(crate::run::PassEnd::Parked {
14699                cause: ParkCause::Upgrade { .. }
14700            })
14701        ));
14702        std::fs::remove_file(index_lock).unwrap();
14703        // No implementer is configured: a persistence retry cannot dispatch a seat.
14704        runner.implement().await.unwrap();
14705        assert!(runner.state.candidate_preservation.is_empty());
14706        assert!(
14707            runner
14708                .state
14709                .candidates
14710                .iter()
14711                .all(|c| c.commits == 1 && !c.empty)
14712        );
14713        assert!(git::is_clean(&wt).await.unwrap());
14714        assert!(
14715            crate::run::read_artifact(&runner.state, "cand-A.patch")
14716                .unwrap()
14717                .contains("later tracked edits")
14718        );
14719    }
14720
14721    #[tokio::test]
14722    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14723        ask_test_home();
14724        let tmp = tempfile::tempdir().unwrap();
14725        let repo = tmp.path().join("repo");
14726        std::fs::create_dir(&repo).unwrap();
14727        init_repo(&repo);
14728        let wt = tmp.path().join("candidate");
14729        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14730            .await
14731            .unwrap();
14732        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14733        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14734        assert!(wt.join("uncommitted").exists());
14735        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14736        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14737        let mut runner = runner_at(RunStatus::Implementing);
14738        runner.state.repo = repo.clone();
14739        runner.state.base_commit = "missing-base".to_owned();
14740        runner.state.candidates.push(Candidate {
14741            index: 0,
14742            label: 'A',
14743            agent: "alpha".into(),
14744            branch: "candidate".into(),
14745            worktree: wt.clone(),
14746            summary: "completed".into(),
14747            stat: String::new(),
14748            files: 0,
14749            commits: 0,
14750            empty: false,
14751            failed: None,
14752            verified_noop: None,
14753            duration_ms: 1,
14754            folded: false,
14755        });
14756        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14757        assert!(
14758            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14759                .await
14760                .is_err()
14761        );
14762        assert!(!runner.state.candidates[0].folded);
14763        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14764        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14765        runner
14766            .state
14767            .candidate_preservation
14768            .insert('A', "pending".into());
14769        assert!(runner.persist_candidates().await.is_err());
14770        assert!(!runner.state.candidates[0].empty);
14771        assert!(!runner.state.candidates[0].folded);
14772        assert!(wt.exists());
14773        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14774        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14775        std::fs::create_dir_all(&patch_path).unwrap();
14776        assert!(
14777            runner.persist_candidates().await.is_err(),
14778            "artifact failure stays pending"
14779        );
14780        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14781        assert!(!runner.state.candidates[0].empty);
14782        std::fs::remove_dir(patch_path).unwrap();
14783        runner.persist_candidates().await.unwrap();
14784        assert!(
14785            runner.state.candidates[0].empty,
14786            "only successful inspection confirms no change"
14787        );
14788    }
14789
14790    fn init_repo(dir: &Path) {
14791        let run = |args: &[&str]| {
14792            let out = std::process::Command::new("git")
14793                .args(args)
14794                .current_dir(dir)
14795                .quiet()
14796                .output()
14797                .expect("spawn git");
14798            assert!(
14799                out.status.success(),
14800                "git {args:?} failed: {}",
14801                String::from_utf8_lossy(&out.stderr)
14802            );
14803        };
14804        run(&["init", "-b", "main"]);
14805        run(&["config", "user.name", "magi test"]);
14806        run(&["config", "user.email", "magi@example.com"]);
14807        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14808        run(&["add", "-A"]);
14809        run(&["commit", "-m", "init"]);
14810    }
14811
14812    // `settle_questions` is what closes the ghost the phone showed: a run's
14813    // seat asked something, the run then ended, and nothing was left to
14814    // abandon the question it left `open`. `HOME` is a process-wide
14815    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14816    // first time it runs in the binary — every test below still reaches the
14817    // same directory whichever call won, and each gets its own run id from
14818    // `RunState::new`, so they never collide there.
14819    fn ask_test_home() {
14820        crate::run::pin_test_home();
14821    }
14822
14823    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14824    /// reads nothing else off it.
14825    fn runner_at(status: RunStatus) -> Runner {
14826        let mut state = RunState::new(
14827            PathBuf::from("/nonexistent/repo"),
14828            "main".to_owned(),
14829            "deadbeef".to_owned(),
14830            "task".to_owned(),
14831            Config::default(),
14832        );
14833        // These fixtures share one home and save concurrently. Short clock
14834        // IDs can collide, making an ownership test read another test's run.
14835        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14836        state.id = format!(
14837            "20000101-000000-{:04x}",
14838            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14839        );
14840        state.status = status;
14841        Runner {
14842            state,
14843            roles: ResolvedRoles {
14844                implementers: Vec::new(),
14845                judges: Vec::new(),
14846                reviewers: Vec::new(),
14847                fixer: None,
14848                conductor: conductor(),
14849                implementer_roster: Vec::new(),
14850                judge_roster: Vec::new(),
14851                reviewer_roster: Vec::new(),
14852            },
14853            sem: Arc::new(Semaphore::new(1)),
14854            pause: Pause::new(),
14855            interrupt: Pause::new(),
14856        }
14857    }
14858
14859    #[tokio::test]
14860    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14861        crate::run::pin_test_home();
14862        let mut runner = runner_at(RunStatus::Gating);
14863        runner.state.save().unwrap();
14864        let id = runner.state.id.clone();
14865        let passes = runner.state.passes.len();
14866        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14867        let mut claiming = Box::pin(runner.claim_driver());
14868        std::future::poll_fn(|cx| {
14869            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14870            std::task::Poll::Ready(())
14871        })
14872        .await;
14873        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14874        drop(owner);
14875        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14876            .await
14877            .unwrap()
14878            .unwrap();
14879        assert_eq!(execution.runner.state.passes.len(), passes + 1);
14880        assert!(execution.runner.state.resource_block.is_none());
14881        assert_eq!(
14882            execution.runner.state.liveness(false),
14883            crate::run::Liveness::Live
14884        );
14885        drop(execution);
14886        assert!(RunState::load(&id).unwrap().driver_exited);
14887    }
14888
14889    #[tokio::test]
14890    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14891        crate::run::pin_test_home();
14892        let mut runner = runner_at(RunStatus::Gating);
14893        runner.state.save().unwrap();
14894        // run.json can commit, but the subsequent lease metadata write fails.
14895        // Cleanup must run after releasing the short ownership lock.
14896        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14897        assert!(runner.claim_driver().await.is_err());
14898        assert!(runner.state.driver_exited);
14899        assert_eq!(runner.state.driver_pid, None);
14900        let disk = RunState::load(&runner.state.id).unwrap();
14901        assert!(disk.driver_exited);
14902        assert_eq!(disk.driver_pid, None);
14903        assert_eq!(disk.status, RunStatus::Gating);
14904    }
14905
14906    #[tokio::test]
14907    async fn an_early_graph_error_releases_its_driver() {
14908        crate::run::pin_test_home();
14909        let mut runner = runner_at(RunStatus::Blocked);
14910        runner.state.config.graph.selected_implementer = Some("selected".into());
14911        runner.state.save().unwrap();
14912        assert!(runner.execute().await.is_err());
14913        let disk = RunState::load(&runner.state.id).unwrap();
14914        assert!(disk.driver_exited);
14915        assert_eq!(disk.driver_pid, None);
14916        assert_eq!(disk.status, RunStatus::Blocked);
14917    }
14918
14919    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14920        crate::run::MergeRepair {
14921            question: "repair-question".into(),
14922            head: "old-head".into(),
14923            pr: "https://github.com/o/r/pull/42".into(),
14924            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14925            gate: vec![outcome(Some(0), false)],
14926            gate_head: Some("old-head".into()),
14927            gate_fixes: Vec::new(),
14928            pre_gate: Vec::new(),
14929            phase,
14930            failure: None,
14931        }
14932    }
14933
14934    #[tokio::test]
14935    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14936        crate::run::pin_test_home();
14937        let mut runner = runner_at(RunStatus::Landing);
14938        runner.state.merge_repairs.push(merge_repair_request(
14939            crate::run::MergeRepairPhase::Requested,
14940        ));
14941        runner.state.save().unwrap();
14942        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14943        assert!(
14944            runner
14945                .execute()
14946                .await
14947                .unwrap_err()
14948                .to_string()
14949                .contains("already running")
14950        );
14951        assert_eq!(
14952            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14953            crate::run::MergeRepairPhase::Requested
14954        );
14955        drop(claim);
14956        let mut disk = runner.state.clone();
14957        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14958        disk.save().unwrap();
14959        assert!(
14960            runner
14961                .execute()
14962                .await
14963                .unwrap_err()
14964                .to_string()
14965                .contains("decision changed")
14966        );
14967        assert_eq!(
14968            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14969            crate::run::MergeRepairPhase::Complete
14970        );
14971    }
14972
14973    #[test]
14974    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14975        crate::run::pin_test_home();
14976        let mut runner = runner_at(RunStatus::Landing);
14977        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14978        runner.state.reviews = request.reviews.clone();
14979        runner.state.gate = request.gate.clone();
14980        runner.state.gate_ran = true;
14981        runner.state.land_approval = Some(crate::run::LandApproval {
14982            question: "old".into(),
14983            head: "old-head".into(),
14984        });
14985        runner.state.merge_repairs.push(request);
14986        runner.begin_merge_repair_review(0).unwrap();
14987        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14988        assert!(!runner.state.gate_status().ok());
14989        assert!(runner.state.land_approval.is_none());
14990        let disk = RunState::load(&runner.state.id).unwrap();
14991        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14992        assert!(disk.merge_repairs[0].gate[0].ok());
14993        assert_eq!(
14994            disk.merge_repairs[0].phase,
14995            crate::run::MergeRepairPhase::Reviewing
14996        );
14997        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14998        assert!(disk.reviews.is_empty());
14999    }
15000
15001    #[tokio::test]
15002    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
15003        crate::run::pin_test_home();
15004        for phase in [
15005            crate::run::MergeRepairPhase::Fixing,
15006            crate::run::MergeRepairPhase::Requested,
15007        ] {
15008            let mut runner = runner_at(RunStatus::Landing);
15009            runner.state.config.graph.land_rounds = 0;
15010            runner.state.merge_repairs.push(merge_repair_request(phase));
15011            runner.state.save().unwrap();
15012            assert!(!runner.repair_merge_review().await.unwrap());
15013            assert_eq!(runner.state.status, RunStatus::Blocked);
15014            assert_eq!(
15015                runner.state.merge_repairs[0].phase,
15016                crate::run::MergeRepairPhase::Failed
15017            );
15018            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
15019            assert!(
15020                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
15021                    "interrupted"
15022                } else {
15023                    "budget exhausted"
15024                })
15025            );
15026            assert!(runner.state.merge.is_none());
15027        }
15028    }
15029
15030    /// `park_here` folding in the reason `Pause::park_because` recorded -
15031    /// this is what lets an operator reading a run's events tell an
15032    /// interrupt-driven park from an ordinary shutdown park.
15033    #[test]
15034    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15035        crate::run::pin_test_home();
15036        let mut runner = runner_at(RunStatus::Implementing);
15037        let interrupt = Pause::new();
15038        runner.watch_interrupt(interrupt.clone());
15039
15040        interrupt.park_because("task a1b2 asked to run first");
15041
15042        assert!(runner.park_here().expect("park_here"));
15043        assert!(runner.state.parked);
15044        let last = runner.state.events.last().expect("a park event");
15045        assert_eq!(last.node, "park");
15046        assert!(
15047            last.message.contains("task a1b2 asked to run first"),
15048            "expected the interrupt reason in {:?}",
15049            last.message
15050        );
15051    }
15052
15053    /// The cause is stored on the pass when it parks, and the first cause a
15054    /// shared pause was given is the one that stays.
15055    #[test]
15056    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15057        crate::run::pin_test_home();
15058        let mut runner = runner_at(RunStatus::Implementing);
15059        runner.state.begin_pass();
15060        let shutdown = Pause::new();
15061        runner.on_pause(shutdown.clone());
15062        let upgrade = ParkCause::Upgrade {
15063            from: "0.121.0".to_owned(),
15064            to: Some("0.121.1".to_owned()),
15065        };
15066        shutdown.park_for(upgrade.clone());
15067        shutdown.park_for(ParkCause::Operator);
15068
15069        assert!(runner.park_here().expect("park_here"));
15070        assert_eq!(
15071            runner.state.passes[0].end,
15072            Some(crate::run::PassEnd::Parked { cause: upgrade })
15073        );
15074        let last = runner.state.events.last().expect("a park event");
15075        assert!(last.message.contains("0.121.0"), "{}", last.message);
15076    }
15077
15078    #[tokio::test]
15079    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15080        crate::run::pin_test_home();
15081        let mut runner = runner_at(RunStatus::Reviewing);
15082        runner.state.save().unwrap();
15083        let mut execution = runner.claim_driver().await.expect("first execution");
15084        assert_eq!(execution.runner.state.passes.len(), 1);
15085        execution
15086            .driver
15087            .finish(&mut execution.runner.state)
15088            .await
15089            .unwrap();
15090    }
15091
15092    #[tokio::test]
15093    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15094        crate::run::pin_test_home();
15095        let mut runner = runner_at(RunStatus::Reviewing);
15096        runner.state.config.agents = vec![conductor()];
15097        runner.state.config.graph.timeout_resource_wait = 0;
15098        runner.state.save().unwrap();
15099        let id = runner.state.id.clone();
15100        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15101        assert!(runner.claim_driver().await.is_err());
15102        drop(owner);
15103        drop(runner);
15104        let mut resumed = Runner::resume(&id).unwrap();
15105        assert_eq!(resumed.state.driver_claimed, Some(false));
15106        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15107        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15108        let mut execution = resumed
15109            .claim_driver()
15110            .await
15111            .expect("retry initial claim after reload");
15112        assert_eq!(execution.runner.state.driver_claimed, Some(true));
15113        assert_eq!(execution.runner.state.passes.len(), 1);
15114        execution
15115            .driver
15116            .finish(&mut execution.runner.state)
15117            .await
15118            .unwrap();
15119    }
15120
15121    #[tokio::test]
15122    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15123        crate::run::pin_test_home();
15124        let mut runner = runner_at(RunStatus::Reviewing);
15125        runner.state.config.agents = vec![conductor()];
15126        runner.state.save().unwrap();
15127        let id = runner.state.id.clone();
15128        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15129        let mut resumed = Runner::resume(&id).unwrap();
15130        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15131        assert!(resumed.claim_driver().await.is_err());
15132        assert!(RunState::load(&id).unwrap().passes.is_empty());
15133        drop(lease);
15134        let mut execution = resumed.claim_driver().await.unwrap();
15135        execution
15136            .driver
15137            .finish(&mut execution.runner.state)
15138            .await
15139            .unwrap();
15140    }
15141
15142    #[tokio::test]
15143    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15144        crate::run::pin_test_home();
15145        let mut runner = runner_at(RunStatus::Reviewing);
15146        let state = &mut runner.state;
15147        state.config.agents = vec![conductor()];
15148        state.driver_claimed = None;
15149        state.driver_pid = Some(std::process::id());
15150        state.driver_started_at = None;
15151        state.begin_pass();
15152        state.save().unwrap();
15153        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15154        std::fs::write(
15155            state.dir().join("driver.json"),
15156            serde_json::to_vec(&serde_json::json!({
15157                "pid": state.driver_pid, "started_at": null,
15158                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15159            }))
15160            .unwrap(),
15161        )
15162        .unwrap();
15163        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15164        drop(lease);
15165        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15166        let mut resumed = Runner::resume(&state.id).unwrap();
15167        assert!(!resumed.state.driver_exited);
15168        let mut execution = resumed
15169            .claim_driver()
15170            .await
15171            .expect("lease proves previous execution ended");
15172        assert_eq!(execution.runner.state.passes.len(), 2);
15173        execution
15174            .driver
15175            .finish(&mut execution.runner.state)
15176            .await
15177            .unwrap();
15178    }
15179
15180    #[tokio::test]
15181    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15182        crate::run::pin_test_home();
15183        let mut runner = runner_at(RunStatus::Reviewing);
15184        let mut json = serde_json::to_value(&runner.state).unwrap();
15185        json.as_object_mut().unwrap().remove("driver_claimed");
15186        runner.state = serde_json::from_value(json).unwrap();
15187        assert_eq!(runner.state.driver_claimed, None);
15188        runner.state.save().unwrap();
15189        let mut execution = runner
15190            .claim_driver()
15191            .await
15192            .expect("the execution lease, not a missing marker, excludes a second driver");
15193        assert_eq!(execution.runner.state.passes.len(), 1);
15194        execution
15195            .driver
15196            .finish(&mut execution.runner.state)
15197            .await
15198            .unwrap();
15199    }
15200
15201    #[tokio::test]
15202    async fn driver_admission_refuses_a_proven_live_driver() {
15203        crate::run::pin_test_home();
15204        let mut runner = runner_at(RunStatus::Reviewing);
15205        let pid = std::process::id();
15206        runner.state.driver_pid = Some(pid);
15207        runner.state.driver_started_at = crate::proc::process_started_at(pid);
15208        assert!(runner.state.driver_started_at.is_some());
15209        runner.state.save().unwrap();
15210        let error = runner
15211            .claim_driver()
15212            .await
15213            .err()
15214            .expect("a live driver must refuse");
15215        assert!(error.to_string().contains("live driver"), "{error:#}");
15216        let saved = RunState::load(&runner.state.id).unwrap();
15217        assert_eq!(saved.driver_pid, Some(pid));
15218        assert_eq!(saved.status, RunStatus::Reviewing);
15219        assert!(saved.passes.is_empty());
15220    }
15221
15222    #[test]
15223    fn parking_resource_halt_persists_the_pass_once() {
15224        crate::run::pin_test_home();
15225        let mut runner = runner_at(RunStatus::Reviewing);
15226        runner.state.begin_pass();
15227        runner.state.resource_block = Some(crate::run::ResourceBlock {
15228            node: "fix".into(),
15229            kind: "shared build cache".into(),
15230            detail: "cancelled".into(),
15231            at: Timestamp::now(),
15232            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15233        });
15234        let cause = ParkCause::Upgrade {
15235            from: "old".into(),
15236            to: Some("new".into()),
15237        };
15238        runner.pause.park_for(cause.clone());
15239        // Both direct fixer exits and graph resource boundaries must complete
15240        // parking, rather than returning a stale Reviewing record.
15241        runner.stop_for_resource().unwrap();
15242        assert!(runner.resource_halt().unwrap());
15243        assert!(runner.park_here().unwrap());
15244        let saved = RunState::load(&runner.state.id).unwrap();
15245        assert!(saved.parked);
15246        assert_eq!(saved.status, RunStatus::Reviewing);
15247        assert!(saved.passes[0].ended_at.is_some());
15248        assert_eq!(
15249            saved.passes[0].end,
15250            Some(crate::run::PassEnd::Parked {
15251                cause: cause.clone()
15252            })
15253        );
15254        assert_eq!(
15255            saved.resource_block.unwrap().cause,
15256            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15257        );
15258        assert_eq!(
15259            saved
15260                .events
15261                .iter()
15262                .filter(|event| event.node == "park")
15263                .count(),
15264            1
15265        );
15266    }
15267
15268    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
15269    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
15270    /// cleared) must not make a *different* run - one only watching its own,
15271    /// unshared interrupt `Pause` - see itself as parked. If a future change
15272    /// ever collapsed these back into one handle, the interrupt scheduler
15273    /// would park every run for the rest of the daemon's life, not just the
15274    /// one it meant to interrupt.
15275    #[test]
15276    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15277        crate::run::pin_test_home();
15278        let mut runner = runner_at(RunStatus::Implementing);
15279        let shutdown = Pause::new();
15280        runner.on_pause(shutdown.clone());
15281        let interrupt = Pause::new();
15282        runner.watch_interrupt(interrupt.clone());
15283
15284        // Nobody has asked for anything yet.
15285        assert!(!runner.park_here().expect("park_here"));
15286        assert!(!runner.state.parked);
15287
15288        // Only the interrupt handle fires; the shutdown handle stays clear.
15289        interrupt.park_because("test");
15290        assert!(!shutdown.parked());
15291        assert!(runner.park_here().expect("park_here"));
15292    }
15293
15294    /// The property every prior attempt at this feature failed to pin down:
15295    /// asking a run to park while one of its nodes has a real, in-flight
15296    /// async operation running (an agent call, in production) must not cut
15297    /// that operation short. `park_here` is only ever consulted *between*
15298    /// `execute`'s node calls - see its own doc - so nothing inside a node
15299    /// can observe a park request until the node itself returns. This proves
15300    /// that structurally, with real `tokio` concurrency and a channel
15301    /// handshake (never a sleep, which would only prove "usually", not
15302    /// "cannot"): the "node" below reports that it has genuinely started,
15303    /// and only then is the park requested; the node still has to be told to
15304    /// finish before `park_here` is ever called, exactly mirroring every
15305    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
15306    /// in `execute`.
15307    #[tokio::test]
15308    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15309        crate::run::pin_test_home();
15310        let mut runner = runner_at(RunStatus::Implementing);
15311        let interrupt = Pause::new();
15312        runner.watch_interrupt(interrupt.clone());
15313
15314        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15315        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15316
15317        // Stands in for one node's in-flight agent call: it proves it has
15318        // genuinely started, then blocks - exactly as a spawned CLI process
15319        // does - until told to finish.
15320        let node = async move {
15321            started_tx.send(()).expect("send started");
15322            finish_rx.await.expect("recv finish");
15323            "node finished"
15324        };
15325
15326        let interrupter = async move {
15327            started_rx.await.expect("recv started");
15328            // The call is now genuinely in flight. Ask it to park.
15329            interrupt.park_because("higher-priority task waiting");
15330            // Nothing the node does can observe this yet - there is no
15331            // check inside it, by construction - so let the executor run
15332            // anything pending and then let the node finish on its own.
15333            tokio::task::yield_now().await;
15334            finish_tx.send(()).expect("send finish");
15335        };
15336
15337        let (node_result, ()) = tokio::join!(node, interrupter);
15338        assert_eq!(
15339            node_result, "node finished",
15340            "the in-flight call ran to completion"
15341        );
15342
15343        // Only now, at the boundary the real `execute` would check right
15344        // after this node, does the park take effect.
15345        assert!(runner.park_here().expect("park_here"));
15346        assert!(runner.state.parked);
15347    }
15348
15349    /// A run parked mid-competition carries every field it had accumulated
15350    /// through the exact same disk round-trip an ordinary resume uses -
15351    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
15352    /// Nothing about parking for an interrupt is a special case of that path;
15353    /// this is what proves it rather than assuming it.
15354    #[test]
15355    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15356        crate::run::pin_test_home();
15357        let mut runner = runner_at(RunStatus::Judging);
15358        // `Runner::resume` re-resolves roles from the saved config, which
15359        // refuses an empty roster - give it the same minimal one `conductor`
15360        // itself uses.
15361        runner.state.config.agents = vec![conductor()];
15362        runner.state.candidates = vec![Candidate {
15363            index: 0,
15364            label: 'A',
15365            agent: "alpha".to_owned(),
15366            branch: "magi/x/A".to_owned(),
15367            worktree: PathBuf::from("/nonexistent/worktree"),
15368            summary: "did the thing".to_owned(),
15369            stat: "1 file changed".to_owned(),
15370            files: 1,
15371            commits: 1,
15372            empty: false,
15373            failed: None,
15374            verified_noop: None,
15375            duration_ms: 1234,
15376            folded: false,
15377        }];
15378        let run_id = runner.state.id.clone();
15379
15380        let interrupt = Pause::new();
15381        runner.watch_interrupt(interrupt.clone());
15382        interrupt.park_because("task c3d4 asked to run first");
15383        assert!(runner.park_here().expect("park_here"));
15384
15385        let resumed = Runner::resume(&run_id).expect("resume");
15386        assert_eq!(resumed.state.candidates.len(), 1);
15387        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15388        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15389        assert_eq!(resumed.state.status, runner.state.status);
15390        assert!(
15391            resumed.state.parked,
15392            "still parked until `execute` actually walks the graph again"
15393        );
15394        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15395    }
15396
15397    /// A fresh open question on `run`, stored and handed back for assertions.
15398    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15399        let mut q = ask::Question::new(
15400            run.to_owned(),
15401            "implement".to_owned(),
15402            "impl-A".to_owned(),
15403            "Which storage backend should the cache use?".to_owned(),
15404            String::new(),
15405            vec!["SQLite".to_owned(), "Redis".to_owned()],
15406        );
15407        store.put(&mut q).unwrap();
15408        q
15409    }
15410
15411    #[test]
15412    fn a_failed_runs_open_question_is_abandoned() {
15413        ask_test_home();
15414        let store = ask::Questions::open();
15415        let mut runner = runner_at(RunStatus::Failed);
15416        let run = runner.state.id.clone();
15417        let q = ask_open_question(&store, &run);
15418
15419        runner.settle_questions();
15420
15421        let back = store.get(&q.id).unwrap();
15422        assert!(
15423            !back.status.open(),
15424            "the seat that asked died with the run; nobody is left to read an answer"
15425        );
15426        assert!(
15427            back.detail.contains(&run) && back.detail.contains("failed"),
15428            "the reason names what the run became, not just that it is gone: {}",
15429            back.detail
15430        );
15431    }
15432
15433    #[test]
15434    fn a_merged_runs_open_question_is_abandoned_too() {
15435        ask_test_home();
15436        let store = ask::Questions::open();
15437        // A run that finishes cleanly still leaves nobody to read an answer -
15438        // this is not only a failure-path cleanup.
15439        for status in [RunStatus::Merged, RunStatus::Ready] {
15440            let mut runner = runner_at(status);
15441            let run = runner.state.id.clone();
15442            let q = ask_open_question(&store, &run);
15443
15444            runner.settle_questions();
15445
15446            let back = store.get(&q.id).unwrap();
15447            assert!(
15448                !back.status.open(),
15449                "{status:?} run's question must not outlive the run"
15450            );
15451        }
15452    }
15453
15454    #[test]
15455    fn a_still_resumable_runs_open_question_is_left_alone() {
15456        ask_test_home();
15457        let store = ask::Questions::open();
15458        // `Blocked` and `Stalled` can still be resumed — the candidates, the
15459        // review round and the seat sessions are all still on disk — so a
15460        // question asked mid-round may yet get a real answer from a real
15461        // resume. Sweeping it here would be exactly the failure mode this
15462        // whole feature exists to avoid on the other side.
15463        for status in [RunStatus::Blocked, RunStatus::Stalled] {
15464            let mut runner = runner_at(status);
15465            let run = runner.state.id.clone();
15466            let q = ask_open_question(&store, &run);
15467
15468            runner.settle_questions();
15469
15470            let back = store.get(&q.id).unwrap();
15471            assert!(
15472                back.status.open(),
15473                "{status:?} is still alive; the question must still be waiting"
15474            );
15475        }
15476    }
15477
15478    #[test]
15479    fn settle_questions_never_touches_an_already_answered_question() {
15480        ask_test_home();
15481        let store = ask::Questions::open();
15482        let mut runner = runner_at(RunStatus::Failed);
15483        let run = runner.state.id.clone();
15484        let mut q = ask_open_question(&store, &run);
15485        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15486            .unwrap();
15487        store.put(&mut q).unwrap();
15488
15489        // Called twice, the way a crash-recovered daemon reclaim and the
15490        // graph's own cleanup both can for the same run — `abandon_for_run`
15491        // only ever touches what is still open, so this must be inert both
15492        // times, not merely the second.
15493        runner.settle_questions();
15494        runner.settle_questions();
15495
15496        let back = store.get(&q.id).unwrap();
15497        assert_eq!(
15498            back.status,
15499            ask::QuestionStatus::Answered,
15500            "a real answer is a decision on record, never overwritten by a sweep"
15501        );
15502    }
15503
15504    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15505    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15506    /// without merging, whose winner is still the operator's answer to read.
15507    /// Nothing previously called `fold_run` itself with a real `tally`, so
15508    /// this is the first test to pin down the one distinction the whole
15509    /// automatic-fold feature depends on: the winner's worktree and branch
15510    /// must survive, everything else sharing the run's worktree bay - a
15511    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15512    /// second sweep treats every non-winner directory under the bay alike -
15513    /// must not.
15514    #[tokio::test]
15515    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15516        crate::run::pin_test_home();
15517        let tmp = tempfile::tempdir().expect("tempdir");
15518        let repo = tmp.path().join("repo");
15519        std::fs::create_dir_all(&repo).unwrap();
15520        init_repo(&repo);
15521
15522        let mut config = Config::default();
15523        config.graph.worktree_root = Some(tmp.path().join("wt"));
15524
15525        let mut state = RunState::new(
15526            repo.clone(),
15527            "main".to_owned(),
15528            "deadbeef".to_owned(),
15529            "task".to_owned(),
15530            config,
15531        );
15532        let root = state.worktree_root();
15533        let wt_a = root.join("cand-A");
15534        let wt_b = root.join("cand-B");
15535        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15536            .await
15537            .expect("worktree A");
15538        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15539            .await
15540            .expect("worktree B");
15541
15542        state.candidates = vec![
15543            Candidate {
15544                index: 0,
15545                label: 'A',
15546                agent: "alpha".to_owned(),
15547                branch: "magi/x/A".to_owned(),
15548                worktree: wt_a.clone(),
15549                summary: String::new(),
15550                stat: String::new(),
15551                files: 0,
15552                commits: 0,
15553                empty: false,
15554                failed: None,
15555                verified_noop: None,
15556                duration_ms: 0,
15557                folded: false,
15558            },
15559            Candidate {
15560                index: 1,
15561                label: 'B',
15562                agent: "beta".to_owned(),
15563                branch: "magi/x/B".to_owned(),
15564                worktree: wt_b.clone(),
15565                summary: String::new(),
15566                stat: String::new(),
15567                files: 0,
15568                commits: 0,
15569                empty: false,
15570                failed: None,
15571                verified_noop: None,
15572                duration_ms: 0,
15573                folded: false,
15574            },
15575        ];
15576        state.tally = Some(Tally {
15577            first_choice: BTreeMap::from([('A', 1)]),
15578            borda: BTreeMap::new(),
15579            winner: 'A',
15580            rankings: 1,
15581            unanimous_initial: true,
15582            deliberated: false,
15583            changed_votes: 0,
15584            unanimous_final: true,
15585            tie_break: None,
15586            judges: 1,
15587            present: 1,
15588            quorum: 1,
15589            met_quorum: true,
15590            uncontested: None,
15591        });
15592        state.status = RunStatus::Ready;
15593
15594        fold_run(&mut state, false, &crate::run::home())
15595            .await
15596            .expect("fold_run");
15597
15598        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15599        assert!(
15600            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15601            "the unmerged winner's branch survives"
15602        );
15603        assert!(
15604            !state.candidates[0].folded,
15605            "the winner is not marked folded"
15606        );
15607
15608        assert!(!wt_b.exists(), "the loser's worktree is removed");
15609        assert!(
15610            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15611            "the loser's branch is removed"
15612        );
15613        assert!(state.candidates[1].folded, "the loser is marked folded");
15614    }
15615
15616    #[tokio::test]
15617    #[cfg(unix)]
15618    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15619        use std::os::fd::AsRawFd as _;
15620        use std::time::Duration;
15621
15622        let tmp = tempfile::tempdir().unwrap();
15623        let repo = tmp.path().join("repo");
15624        std::fs::create_dir_all(&repo).unwrap();
15625        init_repo(&repo);
15626        let common = repo.join(".git");
15627        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15628        let home = tmp.path().join("home");
15629        let mut config = Config::default();
15630        config.coordination = crate::config::test_coordination();
15631        config.graph.worktree_root = Some(tmp.path().join("wt"));
15632        config.graph.timeout_resource_wait = 1;
15633        let mut state = RunState::new(
15634            repo.clone(),
15635            "main".into(),
15636            "deadbeef".into(),
15637            "task".into(),
15638            config,
15639        );
15640        let root = state.worktree_root();
15641        let candidate = root.join("cand-A");
15642        let reviewer = root.join("review-1");
15643        let branch = state.branch_for('A');
15644        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15645            .await
15646            .unwrap();
15647        git::worktree_add_detached(&repo, &reviewer, "main")
15648            .await
15649            .unwrap();
15650        git::acquire_worktree_config(&repo).await.unwrap();
15651        state.enabled_worktree_config = true;
15652        state.status = RunStatus::Ready;
15653        state.candidates.push(Candidate {
15654            index: 0,
15655            label: 'A',
15656            agent: "implement".into(),
15657            branch: branch.clone(),
15658            worktree: candidate.clone(),
15659            summary: String::new(),
15660            stat: String::new(),
15661            files: 0,
15662            commits: 0,
15663            empty: false,
15664            failed: None,
15665            verified_noop: None,
15666            duration_ms: 0,
15667            folded: false,
15668        });
15669        state.save_under(&home).unwrap();
15670        let config_before = std::fs::read(common.join("config")).unwrap();
15671        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15672            .await
15673            .unwrap();
15674        // A separate OS handle bypasses the launcher's local gate, exactly as
15675        // an external updater does, and must still prevent all folding work.
15676        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15677        assert_eq!(
15678            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15679            0
15680        );
15681        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15682        assert!(
15683            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15684                .await
15685                .is_err()
15686        );
15687        assert!(candidate.exists());
15688        assert!(reviewer.exists());
15689        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15690        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15691        assert_eq!(
15692            git::git(&repo, &["worktree", "list", "--porcelain"])
15693                .await
15694                .unwrap(),
15695            worktrees_before
15696        );
15697        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15698            .await
15699            .unwrap()
15700            .unwrap_err();
15701        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15702        drop(fold);
15703        assert!(candidate.exists() && reviewer.exists());
15704        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15705        assert_eq!(
15706            state.resource_block.as_ref().unwrap().kind,
15707            "repository coordination"
15708        );
15709        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15710        drop(external);
15711        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15712        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15713            .await
15714            .unwrap()
15715            .unwrap();
15716        drop(fold);
15717        assert!(removed.contains(&branch));
15718        assert!(!root.exists());
15719        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15720        assert!(state.candidates[0].folded);
15721        assert!(!state.enabled_worktree_config);
15722        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15723        let enabled = git::git_raw(
15724            &repo,
15725            &["config", "--local", "--get", "extensions.worktreeConfig"],
15726        )
15727        .await
15728        .unwrap();
15729        assert!(!enabled.ok(), "the last config reference was released");
15730    }
15731
15732    /// A branch handed to a later run is that run's (and its pull request's):
15733    /// folding the run that released it must not delete it.
15734    #[tokio::test]
15735    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15736        let tmp = tempfile::tempdir().expect("tempdir");
15737        let repo = tmp.path().join("repo");
15738        std::fs::create_dir_all(&repo).unwrap();
15739        init_repo(&repo);
15740        let home = tmp.path().join("home");
15741
15742        let mut config = Config::default();
15743        config.graph.worktree_root = Some(tmp.path().join("wt"));
15744        let mut state = RunState::new(
15745            repo.clone(),
15746            "main".to_owned(),
15747            "deadbeef".to_owned(),
15748            "task".to_owned(),
15749            config,
15750        );
15751        // The worktree is already gone (released); the branch survives.
15752        git::git(&repo, &["branch", "magi/x/A", "main"])
15753            .await
15754            .expect("branch");
15755        state.candidates = vec![Candidate {
15756            index: 0,
15757            label: 'A',
15758            agent: "alpha".to_owned(),
15759            branch: "magi/x/A".to_owned(),
15760            worktree: state.worktree_root().join("cand-A"),
15761            summary: String::new(),
15762            stat: String::new(),
15763            files: 0,
15764            commits: 0,
15765            empty: false,
15766            failed: None,
15767            verified_noop: None,
15768            duration_ms: 0,
15769            folded: true,
15770        }];
15771        state.released_to = Some("20260901-000000-new1".to_owned());
15772        state.released_branches = vec!["magi/x/A".to_owned()];
15773
15774        fold_run(&mut state, true, &home).await.expect("fold_run");
15775        state.status = RunStatus::Merged;
15776        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15777        state.merge = Some(crate::run::MergeOutcome {
15778            mode: MergeMode::Pr,
15779            ok: true,
15780            detail: "confirmed merged".into(),
15781            empty: false,
15782        });
15783        fold_run(&mut state, true, &home)
15784            .await
15785            .expect("merged fold");
15786
15787        assert!(
15788            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15789            "the handed-over branch survives a fold"
15790        );
15791    }
15792
15793    /// A winner with nothing ahead of the base is caught before `gh` is ever
15794    /// asked for a pull request, and the message carries what the task's
15795    /// references resolved to.
15796    #[tokio::test]
15797    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15798        let tmp = tempfile::tempdir().expect("tempdir");
15799        let repo = tmp.path().join("repo");
15800        std::fs::create_dir_all(&repo).unwrap();
15801        init_repo(&repo);
15802        let run = |args: &[&str]| {
15803            let out = std::process::Command::new("git")
15804                .quiet()
15805                .args(args)
15806                .current_dir(&repo)
15807                .output()
15808                .expect("spawn git");
15809            assert!(out.status.success(), "git {args:?}");
15810        };
15811        run(&["branch", "magi/x/A"]);
15812        run(&["checkout", "-q", "-b", "magi/x/B"]);
15813        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15814        run(&["add", "-A"]);
15815        run(&["commit", "-q", "-m", "work"]);
15816        run(&["checkout", "-q", "main"]);
15817
15818        // A pull request is compared against the remote's base, so the
15819        // fixture needs one. Before it exists nothing can be read, and the
15820        // answer must be "not empty".
15821        let probe = RunState::new(
15822            repo.clone(),
15823            "main".to_owned(),
15824            "deadbeef".to_owned(),
15825            "task".to_owned(),
15826            Config::default(),
15827        );
15828        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15829        let bare = tmp.path().join("origin.git");
15830        let out = std::process::Command::new("git")
15831            .quiet()
15832            .args(["init", "-q", "--bare"])
15833            .arg(&bare)
15834            .output()
15835            .expect("spawn git");
15836        assert!(out.status.success(), "git init --bare");
15837        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15838        run(&["push", "-q", "origin", "main"]);
15839
15840        let mut state = RunState::new(
15841            repo.clone(),
15842            "main".to_owned(),
15843            "deadbeef".to_owned(),
15844            "task".to_owned(),
15845            Config::default(),
15846        );
15847        state.seeds = vec![refs::Seed {
15848            token: "magi/27b2/A".to_owned(),
15849            kind: refs::SeedKind::Unresolved,
15850            sha: String::new(),
15851            branch: true,
15852            detail: "no branch or commit named magi/27b2/A".to_owned(),
15853        }];
15854
15855        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15856        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15857        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15858        let detail = empty_candidate_detail(&state, "main");
15859        assert!(detail.starts_with("empty candidate"), "{detail}");
15860        assert!(!detail.contains("magi/27b2/A"), "{detail}");
15861        assert!(detail.contains("--start-from"), "{detail}");
15862    }
15863
15864    /// `status == Ready` used to be read as "this is the harmless
15865    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15866    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15867    /// run's PR was closed without merging — and reentering `merge` with
15868    /// `mode` still `Pr` does not know the difference, so it pushed and
15869    /// opened a second pull request. `mode == Local` reproduces the same
15870    /// blind spot without a network call: reentry must not attempt another
15871    /// git merge once this node has already recorded an outcome.
15872    #[tokio::test]
15873    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15874        let tmp = tempfile::tempdir().expect("tempdir");
15875        let repo = tmp.path().join("repo");
15876        std::fs::create_dir_all(&repo).unwrap();
15877        init_repo(&repo);
15878
15879        let mut config = Config::default();
15880        config.merge.mode = MergeMode::Local;
15881
15882        let mut state = RunState::new(
15883            repo.clone(),
15884            "main".to_owned(),
15885            "deadbeef".to_owned(),
15886            "task".to_owned(),
15887            config,
15888        );
15889        state.candidates = vec![Candidate {
15890            index: 0,
15891            label: 'A',
15892            agent: "alpha".to_owned(),
15893            branch: "does-not-exist".to_owned(),
15894            worktree: repo.clone(),
15895            summary: String::new(),
15896            stat: String::new(),
15897            files: 0,
15898            commits: 0,
15899            empty: false,
15900            failed: None,
15901            verified_noop: None,
15902            duration_ms: 0,
15903            folded: false,
15904        }];
15905        state.tally = Some(Tally {
15906            first_choice: BTreeMap::from([('A', 1)]),
15907            borda: BTreeMap::new(),
15908            winner: 'A',
15909            rankings: 1,
15910            unanimous_initial: true,
15911            deliberated: false,
15912            changed_votes: 0,
15913            unanimous_final: true,
15914            tie_break: None,
15915            judges: 0,
15916            present: 0,
15917            quorum: 0,
15918            met_quorum: true,
15919            uncontested: Some("only candidate A produced a change".to_owned()),
15920        });
15921        state.reviews = vec![ReviewRound {
15922            round: 1,
15923            head: "deadbeef".to_owned(),
15924            verified_head: None,
15925            verified_at: None,
15926            reviews: Vec::new(),
15927            e2e: Vec::new(),
15928            fix: None,
15929            blocking: 0,
15930            answered: 0,
15931            expected: 0,
15932            clean: true,
15933            verify_retried: false,
15934            e2e_deferred: false,
15935            e2e_defer_reason: None,
15936            progressed: false,
15937            vote_split: false,
15938            reconsideration: Vec::new(),
15939            verdict: None,
15940        }];
15941        state.gate = vec![CommandOutcome {
15942            command: "test".to_owned(),
15943            code: Some(0),
15944            output_tail: String::new(),
15945            duration_ms: 0,
15946            resource_blocked: false,
15947        }];
15948        state.gate_ran = true;
15949        // Reached its conclusion already — e.g. `land` closing the PR without
15950        // merging it, which (like the honest `MergeMode::None` path) leaves
15951        // `status` at `Ready`. The recorded outcome is what actually marks
15952        // this node done.
15953        state.status = RunStatus::Ready;
15954        state.merge = Some(MergeOutcome {
15955            mode: MergeMode::Local,
15956            ok: false,
15957            detail: "already concluded".to_owned(),
15958            empty: false,
15959        });
15960
15961        let mut runner = Runner {
15962            state,
15963            roles: ResolvedRoles {
15964                implementers: Vec::new(),
15965                judges: Vec::new(),
15966                reviewers: Vec::new(),
15967                fixer: None,
15968                conductor: conductor(),
15969                implementer_roster: Vec::new(),
15970                judge_roster: Vec::new(),
15971                reviewer_roster: Vec::new(),
15972            },
15973            sem: Arc::new(Semaphore::new(1)),
15974            pause: Pause::new(),
15975            interrupt: Pause::new(),
15976        };
15977
15978        runner.merge().await.expect("merge");
15979
15980        assert_eq!(
15981            runner.state.status,
15982            RunStatus::Ready,
15983            "a concluded run's status must not change on reentry"
15984        );
15985        assert_eq!(
15986            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15987            Some("already concluded"),
15988            "merge must not run again once the node already recorded an outcome"
15989        );
15990    }
15991
15992    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15993    /// when its last attempt was resource-blocked (the shared build cache
15994    /// could not be acquired or confirmed fresh in time - see
15995    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15996    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15997    /// a run merge a tree the gate never actually checked - exactly the case
15998    /// a contended cache produces on every retry until it clears. `merge`
15999    /// must refuse until `gate` has actually recorded an attempt.
16000    #[tokio::test]
16001    async fn merge_refuses_a_gate_that_has_not_actually_run() {
16002        let tmp = tempfile::tempdir().expect("tempdir");
16003        let repo = tmp.path().join("repo");
16004        std::fs::create_dir_all(&repo).unwrap();
16005        init_repo(&repo);
16006
16007        let mut config = Config::default();
16008        config.merge.mode = MergeMode::Local;
16009
16010        let mut state = RunState::new(
16011            repo.clone(),
16012            "main".to_owned(),
16013            "deadbeef".to_owned(),
16014            "task".to_owned(),
16015            config,
16016        );
16017        state.candidates = vec![Candidate {
16018            index: 0,
16019            label: 'A',
16020            agent: "alpha".to_owned(),
16021            branch: "does-not-exist".to_owned(),
16022            worktree: repo.clone(),
16023            summary: String::new(),
16024            stat: String::new(),
16025            files: 0,
16026            commits: 0,
16027            empty: false,
16028            failed: None,
16029            verified_noop: None,
16030            duration_ms: 0,
16031            folded: false,
16032        }];
16033        state.tally = Some(Tally {
16034            first_choice: BTreeMap::from([('A', 1)]),
16035            borda: BTreeMap::new(),
16036            winner: 'A',
16037            rankings: 1,
16038            unanimous_initial: true,
16039            deliberated: false,
16040            changed_votes: 0,
16041            unanimous_final: true,
16042            tie_break: None,
16043            judges: 0,
16044            present: 0,
16045            quorum: 0,
16046            met_quorum: true,
16047            uncontested: Some("only candidate A produced a change".to_owned()),
16048        });
16049        state.reviews = vec![ReviewRound {
16050            round: 1,
16051            head: "deadbeef".to_owned(),
16052            verified_head: None,
16053            verified_at: None,
16054            reviews: Vec::new(),
16055            e2e: Vec::new(),
16056            fix: None,
16057            blocking: 0,
16058            answered: 0,
16059            expected: 0,
16060            clean: true,
16061            verify_retried: false,
16062            e2e_deferred: false,
16063            e2e_defer_reason: None,
16064            progressed: false,
16065            vote_split: false,
16066            reconsideration: Vec::new(),
16067            verdict: None,
16068        }];
16069        // The point: `gate` has not recorded anything yet.
16070        state.gate = Vec::new();
16071        state.gate_ran = false;
16072        state.status = RunStatus::Gating;
16073
16074        let mut runner = Runner {
16075            state,
16076            roles: ResolvedRoles {
16077                implementers: Vec::new(),
16078                judges: Vec::new(),
16079                reviewers: Vec::new(),
16080                fixer: None,
16081                conductor: conductor(),
16082                implementer_roster: Vec::new(),
16083                judge_roster: Vec::new(),
16084                reviewer_roster: Vec::new(),
16085            },
16086            sem: Arc::new(Semaphore::new(1)),
16087            pause: Pause::new(),
16088            interrupt: Pause::new(),
16089        };
16090
16091        runner.merge().await.expect("merge");
16092
16093        assert!(
16094            runner.state.merge.is_none(),
16095            "an empty gate must never be read as a passing one: {:?}",
16096            runner.state.merge
16097        );
16098    }
16099
16100    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
16101    /// commands configured and `merge.mode` is `none` (a review-only run).
16102    /// `gate` must still record a real attempt — zero commands, vacuously
16103    /// passed — rather than leaving `state.gate` empty in a way `merge`
16104    /// cannot tell apart from "never ran"; otherwise the run reaches
16105    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
16106    #[tokio::test]
16107    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16108        let tmp = tempfile::tempdir().expect("tempdir");
16109        let repo = tmp.path().join("repo");
16110        std::fs::create_dir_all(&repo).unwrap();
16111        init_repo(&repo);
16112
16113        // Default config: `verify.gate` empty, `merge.mode` is `none`.
16114        let config = Config::default();
16115
16116        let mut state = RunState::new(
16117            repo.clone(),
16118            "main".to_owned(),
16119            "deadbeef".to_owned(),
16120            "task".to_owned(),
16121            config,
16122        );
16123        state.candidates = vec![Candidate {
16124            index: 0,
16125            label: 'A',
16126            agent: "alpha".to_owned(),
16127            branch: "does-not-exist".to_owned(),
16128            worktree: repo.clone(),
16129            summary: String::new(),
16130            stat: String::new(),
16131            files: 0,
16132            commits: 0,
16133            empty: false,
16134            failed: None,
16135            verified_noop: None,
16136            duration_ms: 0,
16137            folded: false,
16138        }];
16139        state.tally = Some(Tally {
16140            first_choice: BTreeMap::from([('A', 1)]),
16141            borda: BTreeMap::new(),
16142            winner: 'A',
16143            rankings: 1,
16144            unanimous_initial: true,
16145            deliberated: false,
16146            changed_votes: 0,
16147            unanimous_final: true,
16148            tie_break: None,
16149            judges: 0,
16150            present: 0,
16151            quorum: 0,
16152            met_quorum: true,
16153            uncontested: Some("only candidate A produced a change".to_owned()),
16154        });
16155        state.reviews = vec![ReviewRound {
16156            round: 1,
16157            head: "deadbeef".to_owned(),
16158            verified_head: None,
16159            verified_at: None,
16160            reviews: Vec::new(),
16161            e2e: Vec::new(),
16162            fix: None,
16163            blocking: 0,
16164            answered: 0,
16165            expected: 0,
16166            clean: true,
16167            verify_retried: false,
16168            e2e_deferred: false,
16169            e2e_defer_reason: None,
16170            progressed: false,
16171            vote_split: false,
16172            reconsideration: Vec::new(),
16173            verdict: None,
16174        }];
16175
16176        let mut runner = Runner {
16177            state,
16178            roles: ResolvedRoles {
16179                implementers: Vec::new(),
16180                judges: Vec::new(),
16181                reviewers: Vec::new(),
16182                fixer: None,
16183                conductor: conductor(),
16184                implementer_roster: Vec::new(),
16185                judge_roster: Vec::new(),
16186                reviewer_roster: Vec::new(),
16187            },
16188            sem: Arc::new(Semaphore::new(1)),
16189            pause: Pause::new(),
16190            interrupt: Pause::new(),
16191        };
16192
16193        runner.gate().await.expect("gate");
16194        assert!(
16195            runner.state.gate_ran,
16196            "zero configured commands is still a real attempt, not an unrun gate"
16197        );
16198        assert!(runner.state.gate.is_empty());
16199        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16200        assert_ne!(
16201            runner.state.status,
16202            RunStatus::Blocked,
16203            "a gate with nothing to check must not read as failed"
16204        );
16205
16206        runner.merge().await.expect("merge");
16207        assert_eq!(
16208            runner.state.status,
16209            RunStatus::Ready,
16210            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16211        );
16212    }
16213
16214    /// `Config::cache_dir` is derived from `verify.e2e` as well as
16215    /// `verify.gate` (so the e2e leg and the final gate never build against
16216    /// different directories). With zero `verify.gate` commands but a
16217    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
16218    /// that lease before discovering it had nothing to run - so a repo with
16219    /// no gate commands could come back `resource_blocked` (and therefore
16220    /// still `gate_ran == false`) on nothing but an unrelated run holding the
16221    /// cache, exactly the contention this run's own zero commands could
16222    /// never have touched. `gate` must recognise there is nothing to check
16223    /// before it ever asks for the lease.
16224    #[tokio::test]
16225    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16226        crate::run::pin_test_home();
16227        let home = crate::run::home();
16228
16229        let tmp = tempfile::tempdir().expect("tempdir");
16230        let repo = tmp.path().join("repo");
16231        std::fs::create_dir_all(&repo).unwrap();
16232        init_repo(&repo);
16233        // Unique to this test, so holding its lease cannot collide with
16234        // another test sharing the same process-wide `home`.
16235        let cache_dir = tmp.path().join("target");
16236
16237        let mut config = Config::default();
16238        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16239        // `verify.gate` stays empty (the default). Bounded so a regression
16240        // that does start waiting fails the test in seconds, not hangs it.
16241        config.graph.timeout_verify = Some(2);
16242        config.graph.timeout_resource_wait = 2;
16243
16244        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16245        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16246            .expect("no io error acquiring directly")
16247        {
16248            crate::cache::AcquireOutcome::Acquired(g) => g,
16249            crate::cache::AcquireOutcome::Busy(b) => {
16250                panic!("expected the direct acquire to win the lease first: {b:?}")
16251            }
16252        };
16253
16254        let mut state = RunState::new(
16255            repo.clone(),
16256            "main".to_owned(),
16257            "deadbeef".to_owned(),
16258            "task".to_owned(),
16259            config,
16260        );
16261        state.candidates = vec![Candidate {
16262            index: 0,
16263            label: 'A',
16264            agent: "alpha".to_owned(),
16265            branch: "does-not-exist".to_owned(),
16266            worktree: repo.clone(),
16267            summary: String::new(),
16268            stat: String::new(),
16269            files: 0,
16270            commits: 0,
16271            empty: false,
16272            failed: None,
16273            verified_noop: None,
16274            duration_ms: 0,
16275            folded: false,
16276        }];
16277        state.tally = Some(Tally {
16278            first_choice: BTreeMap::from([('A', 1)]),
16279            borda: BTreeMap::new(),
16280            winner: 'A',
16281            rankings: 1,
16282            unanimous_initial: true,
16283            deliberated: false,
16284            changed_votes: 0,
16285            unanimous_final: true,
16286            tie_break: None,
16287            judges: 0,
16288            present: 0,
16289            quorum: 0,
16290            met_quorum: true,
16291            uncontested: Some("only candidate A produced a change".to_owned()),
16292        });
16293        state.reviews = vec![ReviewRound {
16294            round: 1,
16295            head: "deadbeef".to_owned(),
16296            verified_head: None,
16297            verified_at: None,
16298            reviews: Vec::new(),
16299            e2e: Vec::new(),
16300            fix: None,
16301            blocking: 0,
16302            answered: 0,
16303            expected: 0,
16304            clean: true,
16305            verify_retried: false,
16306            e2e_deferred: false,
16307            e2e_defer_reason: None,
16308            progressed: false,
16309            vote_split: false,
16310            reconsideration: Vec::new(),
16311            verdict: None,
16312        }];
16313
16314        let mut runner = Runner {
16315            state,
16316            roles: ResolvedRoles {
16317                implementers: Vec::new(),
16318                judges: Vec::new(),
16319                reviewers: Vec::new(),
16320                fixer: None,
16321                conductor: conductor(),
16322                implementer_roster: Vec::new(),
16323                judge_roster: Vec::new(),
16324                reviewer_roster: Vec::new(),
16325            },
16326            sem: Arc::new(Semaphore::new(1)),
16327            pause: Pause::new(),
16328            interrupt: Pause::new(),
16329        };
16330
16331        let started = std::time::Instant::now();
16332        runner.gate().await.expect("gate");
16333        assert!(
16334            started.elapsed() < Duration::from_secs(1),
16335            "a gate with nothing to run must never wait on a lease it never needed"
16336        );
16337        assert!(
16338            runner.state.gate_ran,
16339            "zero commands is still a real, immediate attempt"
16340        );
16341        assert!(runner.state.gate.is_empty());
16342        assert_ne!(
16343            runner.state.status,
16344            RunStatus::Blocked,
16345            "must not read as resource-blocked on a lease it never asked for"
16346        );
16347    }
16348
16349    /// A separate process carries a live worker's environment throughout all
16350    /// verification entry points; the parallel harness never changes its env.
16351    #[cfg(unix)]
16352    #[tokio::test]
16353    async fn verification_is_standalone_inside_a_live_worker() {
16354        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
16355        if std::env::var_os(CHILD).is_none() {
16356            let home = tempfile::tempdir().unwrap();
16357            let generation = crate::worker::pin_current(home.path()).unwrap();
16358            let output = std::process::Command::new(std::env::current_exe().unwrap())
16359                .args([
16360                    "--exact",
16361                    "graph::tests::verification_is_standalone_inside_a_live_worker",
16362                    "--nocapture",
16363                ])
16364                .env(CHILD, "1")
16365                .env("MAGI_HOME", home.path())
16366                .env("MAGI_WORKER_HOME", home.path())
16367                .env("MAGI_WORKER_ID", "abcdef")
16368                .env("MAGI_WORKER_GENERATION", &generation.id)
16369                .env("MAGI_WORKER_EXE", &generation.executable)
16370                .env("MAGI_RUN", "verification-parent")
16371                .env("MAGI_NODE", "gate")
16372                .env("MAGI_SEAT", "worker-fixture")
16373                .env("MAGI_PROCESS_OWNER", "fixture-owner")
16374                .env("MAGI_NO_AUTOUPDATE", "0")
16375                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
16376                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
16377                .env(
16378                    "VERIFY_EXPECT_TOOLCHAIN",
16379                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
16380                )
16381                .env(
16382                    "VERIFY_EXPECT_CACHE",
16383                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
16384                )
16385                .output()
16386                .unwrap();
16387            assert!(
16388                output.status.success()
16389                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16390                "{}\n{}",
16391                String::from_utf8_lossy(&output.stdout),
16392                String::from_utf8_lossy(&output.stderr)
16393            );
16394            return;
16395        }
16396        let home = crate::run::home();
16397        let directory = home.join("workers/jobs/abcdef");
16398        std::fs::create_dir_all(&directory).unwrap();
16399        let job = crate::worker::Job {
16400            id: "abcdef".into(),
16401            sequence: 1,
16402            kind: "task".into(),
16403            key: "task:verification".into(),
16404            payload: serde_json::json!({}),
16405            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
16406            version: Some(env!("CARGO_PKG_VERSION").into()),
16407            status: crate::worker::Status::Running,
16408            result: None,
16409            pid: Some(std::process::id()),
16410            created_at: 0,
16411            finished_at: None,
16412        };
16413        std::fs::write(
16414            directory.join("job.json"),
16415            serde_json::to_vec(&job).unwrap(),
16416        )
16417        .unwrap();
16418        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
16419        let assert_worker = || {
16420            assert_eq!(
16421                crate::worker::current_job(&home).unwrap().unwrap().id,
16422                job.id
16423            );
16424            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
16425            assert_eq!(
16426                crate::worker::helper_executable(&home).unwrap(),
16427                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
16428            );
16429            assert!(
16430                crate::updater::Checker::new(&crate::config::Update {
16431                    mode: crate::config::UpdateMode::Notify,
16432                    interval: None,
16433                    auto_restart: false,
16434                })
16435                .is_none(),
16436                "the driver remains pinned, even while tests are standalone"
16437            );
16438        };
16439        assert_worker();
16440        let repo = home.join("repo");
16441        std::fs::create_dir_all(&repo).unwrap();
16442        init_repo(&repo);
16443        // Keep storage, seat/control context, containment and tool lookup.
16444        // No inherited values (or credentials) are printed on success.
16445        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
16446            test -z "${MAGI_WORKER_GENERATION+x}" &&
16447            test -z "${MAGI_WORKER_HOME+x}" &&
16448            test -z "${MAGI_WORKER_EXE+x}" &&
16449            test "$MAGI_HOME/repo" -ef . &&
16450            test "$MAGI_RUN" = verification-parent &&
16451            test "$MAGI_NODE" = gate &&
16452            test "$MAGI_SEAT" = worker-fixture &&
16453            test -n "$MAGI_PROCESS_OWNER" &&
16454            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
16455            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
16456            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
16457            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
16458        // Negative control: an ordinary child inherits the contamination.
16459        assert!(
16460            !std::process::Command::new("sh")
16461                .args(["-c", probe])
16462                .current_dir(&repo)
16463                .status()
16464                .unwrap()
16465                .success()
16466        );
16467        let unrelated_home = tempfile::tempdir().unwrap();
16468        assert!(
16469            crate::worker::current_job(unrelated_home.path()).is_err(),
16470            "an inherited worker id cannot be resolved in a unit fixture's home"
16471        );
16472        let mut runner = runner_at(RunStatus::Gating);
16473        runner.state.repo = repo.clone();
16474        runner.state.config.disk.min_free_bytes = 0;
16475        runner.state.config.coordination = crate::config::test_coordination();
16476        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16477        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
16478        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
16479        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
16480        let winner = Candidate {
16481            index: 0,
16482            label: 'A',
16483            agent: "fixture".into(),
16484            branch: "main".into(),
16485            worktree: repo.clone(),
16486            summary: String::new(),
16487            stat: String::new(),
16488            files: 0,
16489            commits: 0,
16490            empty: false,
16491            failed: None,
16492            verified_noop: None,
16493            duration_ms: 0,
16494            folded: false,
16495        };
16496        let (e2e, retried, pids) = run_e2e_with_retry(
16497            &mut runner.state,
16498            &["sh".into(), "-c".into()],
16499            &[format!("{probe} && printf initial")],
16500            &repo,
16501            Duration::from_secs(10),
16502            "initial",
16503        )
16504        .await;
16505        assert!(!retried && pids.is_empty());
16506        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
16507        assert_eq!(e2e[0].output_tail, "initial");
16508        let gate = runner.run_gate(&winner).await.unwrap();
16509        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
16510        // An interrupted repair must recheck e2e from saved state. Landing
16511        // reuses that state and must run both pre-gate and gate again.
16512        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
16513            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
16514            passed: false,
16515            outcomes: Vec::new(),
16516        });
16517        runner.state.save().unwrap();
16518        let mut resumed = RunState::load(&runner.state.id).unwrap();
16519        let wait = crate::repo_lock::WaitPolicy::default();
16520        assert!(matches!(
16521            check_gate_repair_e2e(&mut resumed, &winner, &wait)
16522                .await
16523                .unwrap(),
16524            GateFix::Retry
16525        ));
16526        let saved = RunState::load(&resumed.id).unwrap();
16527        let repair = saved.gate_repair_e2e.unwrap();
16528        assert!(repair.passed);
16529        assert_eq!(repair.outcomes.len(), 1);
16530        assert!(repair.outcomes[0].ok());
16531        let landing = run_landing_gate(&mut resumed, &winner, &wait)
16532            .await
16533            .unwrap();
16534        assert_eq!(landing.len(), 1);
16535        assert!(landing[0].ok(), "{landing:?}");
16536        assert_eq!(
16537            std::fs::read_to_string(repo.join("phases")).unwrap(),
16538            "pre\ngate\ne2e\npre\ngate\n"
16539        );
16540        assert_worker();
16541    }
16542
16543    #[cfg(unix)]
16544    #[tokio::test]
16545    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16546        crate::run::pin_test_home();
16547        let tmp = tempfile::tempdir().unwrap();
16548        let mut config = Config::default();
16549        config.disk.min_free_bytes = 0;
16550        let mut state = RunState::new(
16551            tmp.path().to_owned(),
16552            "main".into(),
16553            "base".into(),
16554            "verification evidence".into(),
16555            config,
16556        );
16557        let (outcomes, pids) = run_commands(
16558            &mut state,
16559            "gate",
16560            "gate",
16561            1,
16562            &["sh".into(), "-c".into()],
16563            &[
16564                "printf before-timeout; exec sleep 30".into(),
16565                "kill -TERM $$".into(),
16566                "exit 7".into(),
16567                "printf success".into(),
16568            ],
16569            tmp.path(),
16570            Duration::from_secs(1),
16571        )
16572        .await;
16573        wait_for_timed_out_children_to_die(&pids).await;
16574        assert_eq!(
16575            pids.len(),
16576            1,
16577            "unleased timeout retains its death-check PID"
16578        );
16579        assert_eq!(outcomes.len(), 4);
16580        assert_eq!(outcomes[0].code, None);
16581        assert!(outcomes[0].output_tail.contains("before-timeout"));
16582        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16583        assert_eq!(outcomes[1].code, None);
16584        assert!(outcomes[1].output_tail.contains("command terminated:"));
16585        assert!(!outcomes[1].output_tail.contains("timed out"));
16586        assert_eq!(outcomes[2].code, Some(7));
16587        assert!(!outcomes[2].output_tail.contains("timed out"));
16588        assert_eq!(outcomes[3].code, Some(0));
16589        assert_eq!(outcomes[3].output_tail, "success");
16590        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16591    }
16592
16593    /// The addendum's second gap: a `verify.gate` command running for real
16594    /// wall-clock time had nothing at all to show for it in `active` before
16595    /// `run_commands` learned to record it — a run could sit in `Gating` for
16596    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
16597    /// what was actually happening. Proven with a genuinely still-running
16598    /// command, not just a before/after check on the final state: a poller
16599    /// task reads the same `run.json` `gate()` is writing, the same way the
16600    /// phone or `magi show` would, while the shell command is still blocked
16601    /// on its own release marker.
16602    #[tokio::test]
16603    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16604        crate::run::pin_test_home();
16605
16606        let tmp = tempfile::tempdir().expect("tempdir");
16607        let repo = tmp.path().join("repo");
16608        std::fs::create_dir_all(&repo).unwrap();
16609        init_repo(&repo);
16610
16611        let mut config = Config::default();
16612        config.verify.gate = vec![
16613            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16614             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16615                .to_owned(),
16616        ];
16617
16618        let mut state = RunState::new(
16619            repo.clone(),
16620            "main".to_owned(),
16621            "deadbeef".to_owned(),
16622            "task".to_owned(),
16623            config,
16624        );
16625        let run_id = state.id.clone();
16626        state.candidates = vec![Candidate {
16627            index: 0,
16628            label: 'A',
16629            agent: "alpha".to_owned(),
16630            branch: "does-not-exist".to_owned(),
16631            worktree: repo.clone(),
16632            summary: String::new(),
16633            stat: String::new(),
16634            files: 0,
16635            commits: 0,
16636            empty: false,
16637            failed: None,
16638            verified_noop: None,
16639            duration_ms: 0,
16640            folded: false,
16641        }];
16642        state.tally = Some(Tally {
16643            first_choice: BTreeMap::from([('A', 1)]),
16644            borda: BTreeMap::new(),
16645            winner: 'A',
16646            rankings: 1,
16647            unanimous_initial: true,
16648            deliberated: false,
16649            changed_votes: 0,
16650            unanimous_final: true,
16651            tie_break: None,
16652            judges: 0,
16653            present: 0,
16654            quorum: 0,
16655            met_quorum: true,
16656            uncontested: Some("only candidate A produced a change".to_owned()),
16657        });
16658        state.reviews = vec![ReviewRound {
16659            round: 1,
16660            head: "deadbeef".to_owned(),
16661            verified_head: None,
16662            verified_at: None,
16663            reviews: Vec::new(),
16664            e2e: Vec::new(),
16665            fix: None,
16666            blocking: 0,
16667            answered: 0,
16668            expected: 0,
16669            clean: true,
16670            verify_retried: false,
16671            e2e_deferred: false,
16672            e2e_defer_reason: None,
16673            progressed: false,
16674            vote_split: false,
16675            reconsideration: Vec::new(),
16676            verdict: None,
16677        }];
16678
16679        let mut runner = Runner {
16680            state,
16681            roles: ResolvedRoles {
16682                implementers: Vec::new(),
16683                judges: Vec::new(),
16684                reviewers: Vec::new(),
16685                fixer: None,
16686                conductor: conductor(),
16687                implementer_roster: Vec::new(),
16688                judge_roster: Vec::new(),
16689                reviewer_roster: Vec::new(),
16690            },
16691            sem: Arc::new(Semaphore::new(1)),
16692            pause: Pause::new(),
16693            interrupt: Pause::new(),
16694        };
16695
16696        let started_marker = repo.join("started.marker");
16697        let release_marker = repo.join("release.marker");
16698        let poller = tokio::spawn(async move {
16699            // Bounded so a regression that never records the task entry
16700            // fails this test in seconds instead of hanging the suite —
16701            // the same shape `a_park_requested_while_a_seat_is_mid_call_
16702            // does_not_cut_it_short` uses for the same reason.
16703            for _ in 0..100 {
16704                if started_marker.exists()
16705                    && let Ok(s) = crate::run::RunState::load(&run_id)
16706                    && let Some(a) = s.active.get("gate")
16707                {
16708                    std::fs::write(&release_marker, b"go").expect("release marker");
16709                    return Some(a.clone());
16710                }
16711                tokio::time::sleep(Duration::from_millis(50)).await;
16712            }
16713            None
16714        });
16715
16716        runner.gate().await.expect("gate");
16717        let captured = poller.await.expect("poller task");
16718        let captured = captured.expect(
16719            "the poller never saw a `gate` task entry in run.json while the command was \
16720             still blocked on its own release marker",
16721        );
16722
16723        assert_eq!(captured.task.as_deref(), Some("gate"));
16724        assert_eq!(captured.node, "gate");
16725        assert_eq!(captured.index, Some(1));
16726        assert_eq!(captured.total, Some(1));
16727        assert!(
16728            captured
16729                .command
16730                .as_deref()
16731                .is_some_and(|c| c.contains("started.marker")),
16732            "{captured:?}"
16733        );
16734
16735        assert!(
16736            runner.state.active.is_empty(),
16737            "the entry must be cleared once the command actually finished: {:?}",
16738            runner.state.active
16739        );
16740        assert!(runner.state.gate_ran);
16741        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16742    }
16743
16744    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16745    /// record for `land`; one with only a Minor, or no reject, leaves none.
16746    #[tokio::test]
16747    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16748        use crate::verdict::{Finding, ReviewVote, Severity};
16749        crate::run::pin_test_home();
16750        let tmp = tempfile::tempdir().expect("tempdir");
16751        let repo = tmp.path().join("repo");
16752        std::fs::create_dir_all(&repo).unwrap();
16753        init_repo(&repo);
16754
16755        for (severity, vote, expect) in [
16756            (Severity::Major, ReviewVote::Reject, true),
16757            (Severity::Minor, ReviewVote::Reject, false),
16758            (Severity::Major, ReviewVote::Approve, false),
16759        ] {
16760            let mut round = review_round(false, 1, 1, 1, false, true);
16761            round.reviews = vec![ReviewRecord {
16762                reviewer: 1,
16763                agent: "alpha".to_owned(),
16764                summary: String::new(),
16765                findings: vec![Finding {
16766                    id: "R1-1-1".to_owned(),
16767                    severity,
16768                    file: None,
16769                    line: None,
16770                    title: "t".to_owned(),
16771                    detail: String::new(),
16772                }],
16773                vote: Some(vote),
16774                failed: None,
16775                duration_ms: 0,
16776                attempts: 0,
16777            }];
16778            let mut state = RunState::new(
16779                repo.clone(),
16780                "main".to_owned(),
16781                "deadbeef".to_owned(),
16782                "task".to_owned(),
16783                Config::default(),
16784            );
16785            state.reviews = vec![round];
16786            let mut runner = Runner {
16787                state,
16788                roles: ResolvedRoles {
16789                    implementers: Vec::new(),
16790                    judges: Vec::new(),
16791                    reviewers: Vec::new(),
16792                    fixer: None,
16793                    conductor: conductor(),
16794                    implementer_roster: Vec::new(),
16795                    judge_roster: Vec::new(),
16796                    reviewer_roster: Vec::new(),
16797                },
16798                sem: Arc::new(Semaphore::new(1)),
16799                pause: Pause::new(),
16800                interrupt: Pause::new(),
16801            };
16802            let shell = runner.state.config.shell();
16803            runner
16804                .stop_reviewing("round budget spent", &shell, &repo)
16805                .await
16806                .expect("stop_reviewing");
16807            assert_eq!(runner.state.status, RunStatus::Gating);
16808            assert_eq!(
16809                runner.state.contested_handoff.is_some(),
16810                expect,
16811                "{severity:?} + {vote:?}"
16812            );
16813        }
16814    }
16815
16816    /// The shape the incident this whole fix responds to actually had: the
16817    /// round budget spent, the last round's own e2e blocked on the shared
16818    /// build cache (held here by a live pid — this test process — exactly
16819    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16820    /// without forking a process). `stop_reviewing` must retry it — not
16821    /// silently leave the round looking untouched (the catch-up-only half of
16822    /// the bug), and not read the contention as a red `e2e` and block the
16823    /// run on it (the other half). Called directly, the same way
16824    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16825    /// above exercises `gate`, so this never needs a real cargo build to
16826    /// reach: the lease is never released, so `with_cache_lease` never gets
16827    /// past acquiring it into anything that would need a real workspace.
16828    #[tokio::test]
16829    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16830        crate::run::pin_test_home();
16831        let home = crate::run::home();
16832
16833        let tmp = tempfile::tempdir().expect("tempdir");
16834        let repo = tmp.path().join("repo");
16835        std::fs::create_dir_all(&repo).unwrap();
16836        init_repo(&repo);
16837        let head = crate::git::rev_parse(&repo, "HEAD")
16838            .await
16839            .expect("rev-parse");
16840        // Unique to this test, so holding its lease cannot collide with
16841        // another test sharing the same process-wide `home`.
16842        let cache_dir = tmp.path().join("target");
16843
16844        let mut config = Config::default();
16845        config.verify.e2e = vec![format!(
16846            "CARGO_TARGET_DIR='{}' test -f README.md",
16847            cache_dir.display()
16848        )];
16849        config.graph.review_rounds = 1;
16850        // Bounded so a regression that does start waiting fails the test in
16851        // seconds, not hangs it.
16852        config.graph.timeout_verify = Some(2);
16853        config.graph.timeout_resource_wait = 2;
16854
16855        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16856        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16857            .expect("no io error acquiring directly")
16858        {
16859            crate::cache::AcquireOutcome::Acquired(g) => g,
16860            crate::cache::AcquireOutcome::Busy(b) => {
16861                panic!("expected the direct acquire to win the lease first: {b:?}")
16862            }
16863        };
16864
16865        let mut state = RunState::new(
16866            repo.clone(),
16867            "main".to_owned(),
16868            head.clone(),
16869            "task".to_owned(),
16870            config,
16871        );
16872        state.candidates = vec![Candidate {
16873            index: 0,
16874            label: 'A',
16875            agent: "alpha".to_owned(),
16876            branch: "does-not-exist".to_owned(),
16877            worktree: repo.clone(),
16878            summary: String::new(),
16879            stat: String::new(),
16880            files: 0,
16881            commits: 0,
16882            empty: false,
16883            failed: None,
16884            verified_noop: None,
16885            duration_ms: 0,
16886            folded: false,
16887        }];
16888        state.tally = Some(Tally {
16889            first_choice: BTreeMap::from([('A', 1)]),
16890            borda: BTreeMap::new(),
16891            winner: 'A',
16892            rankings: 1,
16893            unanimous_initial: true,
16894            deliberated: false,
16895            changed_votes: 0,
16896            unanimous_final: true,
16897            tie_break: None,
16898            judges: 0,
16899            present: 0,
16900            quorum: 0,
16901            met_quorum: true,
16902            uncontested: Some("only candidate A produced a change".to_owned()),
16903        });
16904        // The round budget's last round, deferred: `needs_catchup_run`'s
16905        // other trigger. `stop_reviewing`'s retry machinery must treat this
16906        // exactly like a resource-blocked attempt once it actually runs.
16907        state.reviews = vec![ReviewRound {
16908            round: 1,
16909            head: head.clone(),
16910            verified_head: None,
16911            verified_at: None,
16912            reviews: Vec::new(),
16913            e2e: Vec::new(),
16914            fix: None,
16915            blocking: 1,
16916            answered: 1,
16917            expected: 1,
16918            clean: false,
16919            verify_retried: false,
16920            e2e_deferred: true,
16921            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16922            progressed: false,
16923            vote_split: false,
16924            reconsideration: Vec::new(),
16925            verdict: None,
16926        }];
16927
16928        let mut runner = Runner {
16929            state,
16930            roles: ResolvedRoles {
16931                implementers: Vec::new(),
16932                judges: Vec::new(),
16933                reviewers: Vec::new(),
16934                fixer: None,
16935                conductor: conductor(),
16936                implementer_roster: Vec::new(),
16937                judge_roster: Vec::new(),
16938                reviewer_roster: Vec::new(),
16939            },
16940            sem: Arc::new(Semaphore::new(1)),
16941            pause: Pause::new(),
16942            interrupt: Pause::new(),
16943        };
16944
16945        let shell = runner.state.config.shell();
16946        runner
16947            .stop_reviewing("round budget spent", &shell, &repo)
16948            .await
16949            .expect("stop_reviewing");
16950
16951        let last = runner.state.reviews.last().expect("round record");
16952        assert_eq!(
16953            last.e2e_status(),
16954            E2eStatus::ResourceBlocked,
16955            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16956             failed: {last:?}"
16957        );
16958        assert_eq!(
16959            last.verified_head.as_deref(),
16960            Some(head.as_str()),
16961            "which commit this attempt targeted is known even though nothing finished checking \
16962             it"
16963        );
16964        let first_attempt_at = last
16965            .verified_at
16966            .expect("when this attempt ran is known too");
16967        assert_ne!(
16968            runner.state.status,
16969            RunStatus::Blocked,
16970            "contention is evidence about the machine, not the patch — it must not settle the \
16971             run as blocked: {:?}",
16972            runner.state.status
16973        );
16974        assert!(
16975            !runner
16976                .state
16977                .events
16978                .iter()
16979                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16980            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16981            runner.state.events
16982        );
16983
16984        // The cache is still held: a later reentry must retry the same
16985        // round's verification again — not leave it looking exactly as
16986        // untouched as the first blocked attempt, which is indistinguishable
16987        // from never having tried again at all.
16988        runner
16989            .stop_reviewing("round budget spent", &shell, &repo)
16990            .await
16991            .expect("stop_reviewing retry");
16992        assert_eq!(
16993            runner.state.reviews.len(),
16994            1,
16995            "no new round was started: {:?}",
16996            runner.state.reviews
16997        );
16998        let last = runner.state.reviews.last().expect("round record");
16999        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
17000        assert!(
17001            last.verified_at.expect("still known") > first_attempt_at,
17002            "a second reentry must be a fresh attempt, not a stale copy of the first"
17003        );
17004        assert_ne!(runner.state.status, RunStatus::Blocked);
17005
17006        held.release();
17007    }
17008
17009    #[cfg(unix)]
17010    #[tokio::test]
17011    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
17012        ask_test_home();
17013        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
17014            let tmp = tempfile::tempdir().unwrap();
17015            let repo = tmp.path().join("repo");
17016            std::fs::create_dir(&repo).unwrap();
17017            init_repo(&repo);
17018            let run_git = |args: &[&str]| {
17019                let out = std::process::Command::new("git")
17020                    .args(args)
17021                    .current_dir(&repo)
17022                    .output()
17023                    .unwrap();
17024                assert!(
17025                    out.status.success(),
17026                    "{args:?}: {}",
17027                    String::from_utf8_lossy(&out.stderr)
17028                );
17029            };
17030            let origin = tmp.path().join("origin.git");
17031            run_git(&["init", "--bare", origin.to_str().unwrap()]);
17032            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17033            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17034            run_git(&["add", "spec.rs"]);
17035            run_git(&["commit", "-m", "initial spec"]);
17036            run_git(&["push", "origin", "main"]);
17037            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17038            run_git(&["checkout", "-b", "fix/review"]);
17039            std::fs::write(
17040                repo.join("check.rs"),
17041                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17042            )
17043            .unwrap();
17044            run_git(&["add", "check.rs"]);
17045            run_git(&["commit", "-m", "use spec"]);
17046            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17047            run_git(&["checkout", "main"]);
17048            std::fs::write(
17049                repo.join("spec.rs"),
17050                "struct AgentSpec { max_turn_secs: u64 }\n",
17051            )
17052            .unwrap();
17053            run_git(&["add", "spec.rs"]);
17054            run_git(&["commit", "-m", "extend spec"]);
17055            run_git(&["push", "origin", "main"]);
17056            run_git(&["checkout", "fix/review"]);
17057
17058            let mut runner = runner_at(RunStatus::Reviewing);
17059            runner.state.repo = repo.clone();
17060            runner.state.base_commit = base;
17061            runner.state.pr_head = Some("fix/review".into());
17062            runner.state.config.graph.review_rounds = max_rounds;
17063            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17064            runner.state.config.graph.sessions = false;
17065            runner.state.config.disk.min_free_bytes = 0;
17066            runner.state.config.verify.e2e =
17067                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17068            runner.state.candidates = vec![Candidate {
17069                index: 0,
17070                label: 'A',
17071                agent: "fixture".into(),
17072                branch: "fix/review".into(),
17073                worktree: repo.clone(),
17074                summary: String::new(),
17075                stat: String::new(),
17076                files: 1,
17077                commits: 1,
17078                empty: false,
17079                failed: None,
17080                verified_noop: None,
17081                duration_ms: 0,
17082                folded: false,
17083            }];
17084            runner.state.tally = Some(Tally {
17085                first_choice: BTreeMap::new(),
17086                borda: BTreeMap::new(),
17087                winner: 'A',
17088                rankings: 0,
17089                unanimous_initial: true,
17090                deliberated: false,
17091                changed_votes: 0,
17092                unanimous_final: true,
17093                tie_break: None,
17094                judges: 0,
17095                present: 0,
17096                quorum: 0,
17097                met_quorum: true,
17098                uncontested: Some("review-only".into()),
17099            });
17100            let mut last = review_round(false, 0, 1, 1, false, false);
17101            last.head = reviewed.clone();
17102            last.e2e_deferred = true;
17103            last.e2e[0].resource_blocked = true;
17104            last.e2e[0].output_tail = "waiting for the shared build cache".into();
17105            if max_rounds == 1 {
17106                last.e2e.clear();
17107            }
17108            runner.state.reviews = vec![last];
17109            let calls = tmp.path().join("fix-calls");
17110            let mut fixer = spec("repair");
17111            fixer
17112                .env
17113                .insert("FIX_CALLS".into(), calls.display().to_string());
17114            fixer.command = vec!["sh".into(), "-c".into(), r#"
17115                echo fix >> "$FIX_CALLS"
17116                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17117                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17118                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17119                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17120            "#.into()];
17121            let mut reviewer = spec("reviewer");
17122            reviewer.command = vec![
17123                "sh".into(),
17124                "-c".into(),
17125                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17126                    .into(),
17127            ];
17128            runner.roles.fixer = Some(vec![fixer]);
17129            runner.roles.reviewers = vec![reviewer];
17130
17131            runner.sync_to_base().await.unwrap();
17132            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17133            assert_ne!(rebased, reviewed);
17134            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17135            if reentry {
17136                // Simulate a crash after catch-up evidence was saved but before
17137                // dispatching the fixer. No dedicated pending-repair field exists.
17138                let shell = runner.state.config.shell();
17139                runner
17140                    .stop_unresolved_review("deferred verification", &shell, &repo)
17141                    .await
17142                    .unwrap();
17143                assert!(runner.catchup_repairable());
17144                assert!(!calls.exists());
17145                runner.state = RunState::load(&runner.state.id).unwrap();
17146                assert!(runner.catchup_repairable());
17147            }
17148            runner.review_loop().await.unwrap();
17149            let first = &runner.state.reviews[0];
17150            assert_eq!(first.head, reviewed);
17151            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17152            assert!(
17153                first.verify_retried,
17154                "a reproduced build failure gets exactly one retry: {:?}",
17155                first.e2e
17156            );
17157            assert!(first.e2e[0].output_tail.contains("E0063"));
17158            assert!(!runner.catchup_repairable());
17159            if max_rounds == 2 {
17160                assert_eq!(
17161                    runner.state.status,
17162                    RunStatus::Gating,
17163                    "{}",
17164                    crate::report::run(&runner.state)
17165                );
17166                assert_eq!(runner.state.reviews.len(), 2);
17167                assert!(first.fix.as_ref().unwrap().committed);
17168                assert!(runner.state.reviews[1].clean);
17169                assert_eq!(
17170                    runner
17171                        .state
17172                        .reviews
17173                        .iter()
17174                        .filter(|r| r.fix.is_some())
17175                        .count(),
17176                    1
17177                );
17178                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17179                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17180            } else {
17181                assert_eq!(runner.state.status, RunStatus::Blocked);
17182                assert!(first.fix.is_none());
17183                assert!(!calls.exists());
17184                let recovery = crate::recovery::assess(
17185                    "prior",
17186                    Some(&runner.state),
17187                    &runner.landing_base(),
17188                    crate::recovery::Relation::Diverged,
17189                );
17190                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17191                let reasons = recovery.reasons.join("\n");
17192                assert!(
17193                    reasons.contains(&reviewed) && reasons.contains(&rebased),
17194                    "{reasons}"
17195                );
17196                assert!(
17197                    reasons.contains("base sync to")
17198                        && reasons.contains("additional review is required"),
17199                    "{reasons}"
17200                );
17201            }
17202        }
17203    }
17204
17205    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
17206    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
17207    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
17208    /// own top-of-function fast path (`review_conclusion`) correctly reads
17209    /// this shape as `None` rather than guessing `Blocked`, and the loop's
17210    /// own `for` range is empty once the round budget is spent, so
17211    /// `review_loop` must retry the check itself rather than silently doing
17212    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
17213    /// above exercises directly, but through `review_loop`'s own entry point
17214    /// this time, proving the wiring between the two rather than just the
17215    /// retry logic in isolation.
17216    #[tokio::test]
17217    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17218        crate::run::pin_test_home();
17219        let home = crate::run::home();
17220
17221        let tmp = tempfile::tempdir().expect("tempdir");
17222        let repo = tmp.path().join("repo");
17223        std::fs::create_dir_all(&repo).unwrap();
17224        init_repo(&repo);
17225        let head = crate::git::rev_parse(&repo, "HEAD")
17226            .await
17227            .expect("rev-parse");
17228        let cache_dir = tmp.path().join("target");
17229
17230        let mut config = Config::default();
17231        config.verify.e2e = vec![format!(
17232            "CARGO_TARGET_DIR='{}' test -f README.md",
17233            cache_dir.display()
17234        )];
17235        config.graph.review_rounds = 1;
17236        config.graph.timeout_verify = Some(2);
17237        config.graph.timeout_resource_wait = 2;
17238
17239        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17240        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17241            .expect("no io error acquiring directly")
17242        {
17243            crate::cache::AcquireOutcome::Acquired(g) => g,
17244            crate::cache::AcquireOutcome::Busy(b) => {
17245                panic!("expected the direct acquire to win the lease first: {b:?}")
17246            }
17247        };
17248
17249        let mut state = RunState::new(
17250            repo.clone(),
17251            "main".to_owned(),
17252            head.clone(),
17253            "task".to_owned(),
17254            config,
17255        );
17256        state.candidates = vec![Candidate {
17257            index: 0,
17258            label: 'A',
17259            agent: "alpha".to_owned(),
17260            branch: "does-not-exist".to_owned(),
17261            worktree: repo.clone(),
17262            summary: String::new(),
17263            stat: String::new(),
17264            files: 0,
17265            commits: 0,
17266            empty: false,
17267            failed: None,
17268            verified_noop: None,
17269            duration_ms: 0,
17270            folded: false,
17271        }];
17272        state.tally = Some(Tally {
17273            first_choice: BTreeMap::from([('A', 1)]),
17274            borda: BTreeMap::new(),
17275            winner: 'A',
17276            rankings: 1,
17277            unanimous_initial: true,
17278            deliberated: false,
17279            changed_votes: 0,
17280            unanimous_final: true,
17281            tie_break: None,
17282            judges: 0,
17283            present: 0,
17284            quorum: 0,
17285            met_quorum: true,
17286            uncontested: Some("only candidate A produced a change".to_owned()),
17287        });
17288        // The exact shape a prior process's `stop_reviewing` would have left
17289        // on disk: the round budget's last round, a real attempt already
17290        // made and already resource-blocked.
17291        state.reviews = vec![ReviewRound {
17292            round: 1,
17293            head: head.clone(),
17294            verified_head: Some(head.clone()),
17295            verified_at: Some(jiff::Timestamp::now()),
17296            reviews: Vec::new(),
17297            e2e: vec![CommandOutcome {
17298                command: format!(
17299                    "CARGO_TARGET_DIR='{}' test -f README.md",
17300                    cache_dir.display()
17301                ),
17302                code: None,
17303                output_tail: "waiting for the shared build cache".to_owned(),
17304                duration_ms: 0,
17305                resource_blocked: true,
17306            }],
17307            fix: None,
17308            blocking: 1,
17309            answered: 1,
17310            expected: 1,
17311            clean: false,
17312            verify_retried: false,
17313            e2e_deferred: false,
17314            e2e_defer_reason: None,
17315            progressed: false,
17316            vote_split: false,
17317            reconsideration: Vec::new(),
17318            verdict: None,
17319        }];
17320
17321        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17322        let mut runner = Runner {
17323            state,
17324            roles: ResolvedRoles {
17325                implementers: Vec::new(),
17326                judges: Vec::new(),
17327                reviewers: Vec::new(),
17328                fixer: None,
17329                conductor: conductor(),
17330                implementer_roster: Vec::new(),
17331                judge_roster: Vec::new(),
17332                reviewer_roster: Vec::new(),
17333            },
17334            sem: Arc::new(Semaphore::new(1)),
17335            pause: Pause::new(),
17336            interrupt: Pause::new(),
17337        };
17338
17339        // The lease is still held throughout, so this reentry's own retry is
17340        // also contended — proving `review_loop` actually tried again (not
17341        // that it happened to succeed) is what the timestamp comparison
17342        // below is for.
17343        runner.review_loop().await.expect("review_loop");
17344
17345        assert_eq!(
17346            runner.state.reviews.len(),
17347            1,
17348            "no new round was started on top of the unresolved one: {:?}",
17349            runner.state.reviews
17350        );
17351        let last = &runner.state.reviews[0];
17352        assert_eq!(
17353            last.e2e_status(),
17354            E2eStatus::ResourceBlocked,
17355            "still contended: {last:?}"
17356        );
17357        assert!(
17358            last.verified_at.expect("still known") > first_attempt_at,
17359            "review_loop must have actually retried the check, not left it exactly as found"
17360        );
17361        assert_ne!(
17362            runner.state.status,
17363            RunStatus::Blocked,
17364            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17365            runner.state.status
17366        );
17367
17368        held.release();
17369    }
17370
17371    #[tokio::test]
17372    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17373        crate::run::pin_test_home();
17374        for status in [RunStatus::Blocked, RunStatus::Gating] {
17375            let mut runner = runner_at(status);
17376            runner.state.config.graph.land = true;
17377            runner.state.gate_ran = true;
17378            runner.state.merge = Some(MergeOutcome {
17379                mode: MergeMode::Pr,
17380                ok: false,
17381                detail: "the fixer produced no commit".into(),
17382                empty: false,
17383            });
17384            let url = "https://example.invalid/o/r/pull/1";
17385            runner.state.pr = Some(crate::run::PrRecord {
17386                url: url.into(),
17387                number: 1,
17388                state: "open".into(),
17389                checks: "red".into(),
17390                round: 1,
17391                rounds: 1,
17392                red_at_merge: Vec::new(),
17393            });
17394            runner.state.save().unwrap();
17395            // The nonexistent repository makes observation fail before a forge
17396            // command can run. We still must reach land, rather than silently
17397            // return from the graph with its old gate and merge records.
17398            runner.execute_graph().await.unwrap();
17399            assert_eq!(runner.state.status, RunStatus::Blocked);
17400            assert!(runner.state.land_sync_pending);
17401            assert!(runner.state.events.iter().any(|event|
17402                event.node == "land" && event.message == format!("watching {url}")));
17403            assert_eq!(
17404                runner.state.merge.as_ref().unwrap().detail,
17405                "the fixer produced no commit"
17406            );
17407        }
17408    }
17409
17410    #[tokio::test]
17411    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17412        crate::run::pin_test_home();
17413        let tmp = tempfile::tempdir().expect("tempdir");
17414        let repo = tmp.path().join("repo");
17415        std::fs::create_dir_all(&repo).unwrap();
17416        init_repo(&repo);
17417
17418        let mut config = Config::default();
17419        config.merge.mode = MergeMode::Pr;
17420        config.graph.land = true;
17421        config.graph.land_approval = false;
17422
17423        let mut state = RunState::new(
17424            repo.clone(),
17425            "main".to_owned(),
17426            "deadbeef".to_owned(),
17427            "task".to_owned(),
17428            config,
17429        );
17430        state.candidates = vec![Candidate {
17431            index: 0,
17432            label: 'A',
17433            agent: "alpha".to_owned(),
17434            branch: "does-not-exist".to_owned(),
17435            worktree: repo.clone(),
17436            summary: String::new(),
17437            stat: String::new(),
17438            files: 0,
17439            commits: 0,
17440            empty: false,
17441            failed: None,
17442            verified_noop: None,
17443            duration_ms: 0,
17444            folded: false,
17445        }];
17446        state.tally = Some(Tally {
17447            first_choice: BTreeMap::from([('A', 1)]),
17448            borda: BTreeMap::new(),
17449            winner: 'A',
17450            rankings: 1,
17451            unanimous_initial: true,
17452            deliberated: false,
17453            changed_votes: 0,
17454            unanimous_final: true,
17455            tie_break: None,
17456            judges: 0,
17457            present: 0,
17458            quorum: 0,
17459            met_quorum: true,
17460            uncontested: Some("only candidate A produced a change".to_owned()),
17461        });
17462        state.reviews = vec![ReviewRound {
17463            round: 1,
17464            head: "deadbeef".to_owned(),
17465            verified_head: None,
17466            verified_at: None,
17467            reviews: Vec::new(),
17468            e2e: Vec::new(),
17469            fix: None,
17470            blocking: 0,
17471            answered: 0,
17472            expected: 0,
17473            clean: true,
17474            verify_retried: false,
17475            e2e_deferred: false,
17476            e2e_defer_reason: None,
17477            progressed: false,
17478            vote_split: false,
17479            reconsideration: Vec::new(),
17480            verdict: None,
17481        }];
17482        state.gate = vec![CommandOutcome {
17483            command: "test".to_owned(),
17484            code: Some(0),
17485            output_tail: String::new(),
17486            duration_ms: 0,
17487            resource_blocked: false,
17488        }];
17489        state.gate_ran = true;
17490        // A first pass through `merge` already pushed and opened this pull
17491        // request; `status` is `Landing` because a previous call into `land`
17492        // parked or was interrupted before it reached a terminal outcome.
17493        state.status = RunStatus::Landing;
17494        state.merge = Some(MergeOutcome {
17495            mode: MergeMode::Pr,
17496            ok: true,
17497            detail: "https://example.invalid/x/y/pull/1".to_owned(),
17498            empty: false,
17499        });
17500
17501        // The Landing-resume shortcut calls `run_land` directly rather than
17502        // through `merge`, which is exactly the call site that used to skip
17503        // `settle_questions` - see the fixture below.
17504        ask_test_home();
17505        // A real resume loads a persisted run; driver ownership is checked
17506        // against that record before the Landing shortcut is entered.
17507        state.save().expect("save landing run");
17508        let store = ask::Questions::open();
17509        let q = ask_open_question(&store, &state.id);
17510
17511        let mut runner = Runner {
17512            state,
17513            roles: ResolvedRoles {
17514                implementers: Vec::new(),
17515                judges: Vec::new(),
17516                reviewers: Vec::new(),
17517                fixer: None,
17518                conductor: conductor(),
17519                implementer_roster: Vec::new(),
17520                judge_roster: Vec::new(),
17521                reviewer_roster: Vec::new(),
17522            },
17523            sem: Arc::new(Semaphore::new(1)),
17524            pause: Pause::new(),
17525            interrupt: Pause::new(),
17526        };
17527
17528        // `execute`, not `merge` directly: the Landing-resume shortcut lives
17529        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
17530        // exactly because `review_loop` would otherwise clobber the marker
17531        // first.
17532        runner.execute().await.expect("execute");
17533
17534        assert_eq!(
17535            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17536            Some("https://example.invalid/x/y/pull/1"),
17537            "reentry must not push again or open a second pull request over the \
17538             one `land` is already watching"
17539        );
17540        assert_ne!(
17541            runner.state.status,
17542            RunStatus::Landing,
17543            "land could not actually reach the fake pull request, so it must \
17544             have given up rather than left the run silently parked forever"
17545        );
17546        // `land` could not reach the fake pull request, so it gave up into
17547        // `Blocked` - still resumable, so the question must not have been
17548        // swept just because this branch now also calls `settle_questions`.
17549        assert_eq!(runner.state.status, RunStatus::Blocked);
17550        assert!(
17551            store.get(&q.id).unwrap().status.open(),
17552            "Blocked is still alive; settle_questions must have been a no-op here"
17553        );
17554    }
17555
17556    fn state_with_round(round: ReviewRound) -> RunState {
17557        let mut s = RunState::new(
17558            PathBuf::from("/repo"),
17559            "main".to_owned(),
17560            "abc1234".to_owned(),
17561            "add retries".to_owned(),
17562            Config::default(),
17563        );
17564        s.reviews = vec![round];
17565        s
17566    }
17567
17568    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17569        crate::verdict::Finding {
17570            id: id.to_owned(),
17571            severity,
17572            file: None,
17573            line: None,
17574            title: title.to_owned(),
17575            detail: String::new(),
17576        }
17577    }
17578
17579    #[test]
17580    fn pr_body_names_open_findings_and_declined_ones() {
17581        let round = ReviewRound {
17582            round: 2,
17583            head: "deadbee".to_owned(),
17584            verified_head: None,
17585            verified_at: None,
17586            reviews: vec![ReviewRecord {
17587                attempts: 0,
17588                reviewer: 1,
17589                agent: "alpha".to_owned(),
17590                summary: String::new(),
17591                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17592                vote: None,
17593                failed: None,
17594                duration_ms: 0,
17595            }],
17596            e2e: vec![CommandOutcome {
17597                command: "cargo test".to_owned(),
17598                code: Some(0),
17599                output_tail: String::new(),
17600                duration_ms: 0,
17601                resource_blocked: false,
17602            }],
17603            verify_retried: false,
17604            e2e_deferred: false,
17605            e2e_defer_reason: None,
17606            fix: Some(FixRecord {
17607                agent: "alpha".to_owned(),
17608                addressed: Vec::new(),
17609                rejected: vec![crate::verdict::Rejection {
17610                    id: "R1-1-1".to_owned(),
17611                    why: "not reachable from any caller".to_owned(),
17612                }],
17613                notes: String::new(),
17614                committed: true,
17615                failed: None,
17616                duration_ms: 0,
17617                continuation: None,
17618            }),
17619            blocking: 0,
17620            answered: 1,
17621            expected: 1,
17622            clean: false,
17623            progressed: true,
17624            vote_split: false,
17625            reconsideration: Vec::new(),
17626            verdict: None,
17627        };
17628        let state = state_with_round(round);
17629        let body = pr_message(&state, 'A').body;
17630
17631        assert!(body.contains("add retries"), "the task must still be there");
17632        assert!(body.contains("R2-1-1"), "{body}");
17633        assert!(!body.contains("unused import"), "{body}");
17634        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17635        assert!(
17636            !body.contains("not reachable from any caller"),
17637            "the reason it was declined: {body}"
17638        );
17639    }
17640
17641    #[test]
17642    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17643        let round = ReviewRound {
17644            round: 1,
17645            head: "deadbee".to_owned(),
17646            verified_head: None,
17647            verified_at: None,
17648            reviews: vec![ReviewRecord {
17649                attempts: 0,
17650                reviewer: 1,
17651                agent: "alpha".to_owned(),
17652                summary: String::new(),
17653                findings: Vec::new(),
17654                vote: None,
17655                failed: None,
17656                duration_ms: 0,
17657            }],
17658            e2e: Vec::new(),
17659            verify_retried: false,
17660            e2e_deferred: false,
17661            e2e_defer_reason: None,
17662            fix: None,
17663            blocking: 0,
17664            answered: 1,
17665            expected: 1,
17666            clean: true,
17667            progressed: false,
17668            vote_split: false,
17669            reconsideration: Vec::new(),
17670            verdict: None,
17671        };
17672        let state = state_with_round(round);
17673        let body = pr_message(&state, 'A').body;
17674        assert!(!body.contains("Open review findings"), "{body}");
17675        assert!(!body.contains("Declined"), "{body}");
17676    }
17677
17678    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17679        let mut state = RunState::new(
17680            PathBuf::from("/repo"),
17681            "main".to_owned(),
17682            "abc1234".to_owned(),
17683            instruction.to_owned(),
17684            Config::default(),
17685        );
17686        state.candidates.push(Candidate {
17687            index: 0,
17688            label: 'A',
17689            agent: "alpha".to_owned(),
17690            branch: "magi/x/A".to_owned(),
17691            worktree: PathBuf::from("/wt"),
17692            summary: summary.to_owned(),
17693            stat: String::new(),
17694            files: 1,
17695            commits: 1,
17696            empty: false,
17697            failed: None,
17698            verified_noop: None,
17699            folded: false,
17700            duration_ms: 0,
17701        });
17702        state
17703    }
17704
17705    fn review_state(subjects: &[&str]) -> RunState {
17706        let mut state = state_with_summary(
17707            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17708            "",
17709        );
17710        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17711        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17712        state
17713    }
17714
17715    /// A branch rebased onto a main that moved past the recorded
17716    /// `base_commit` is titled from its own first commit, never main's.
17717    #[tokio::test]
17718    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17719        ask_test_home();
17720        let tmp = tempfile::tempdir().unwrap();
17721        let repo = tmp.path().join("repo");
17722        std::fs::create_dir_all(&repo).unwrap();
17723        init_repo(&repo);
17724        let origin = tmp.path().join("origin.git");
17725        let g = |dir: &Path, args: &[&str]| {
17726            let out = std::process::Command::new("git")
17727                .args(args)
17728                .current_dir(dir)
17729                .quiet()
17730                .output()
17731                .expect("spawn git");
17732            assert!(
17733                out.status.success(),
17734                "git {args:?}: {}",
17735                String::from_utf8_lossy(&out.stderr)
17736            );
17737        };
17738        g(
17739            tmp.path(),
17740            &[
17741                "clone",
17742                "--bare",
17743                "-q",
17744                repo.to_str().unwrap(),
17745                origin.to_str().unwrap(),
17746            ],
17747        );
17748        g(
17749            &repo,
17750            &["remote", "add", "origin", origin.to_str().unwrap()],
17751        );
17752        let c1 = git::rev_parse(&repo, "main").await.unwrap();
17753
17754        // Main moves on; the branch is built on top of the new main.
17755        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17756        g(&repo, &["add", "-A"]);
17757        g(
17758            &repo,
17759            &["commit", "-q", "-m", "chore(deps): update a crate"],
17760        );
17761        g(&repo, &["push", "-q", "origin", "main"]);
17762        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17763        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17764        g(&repo, &["add", "-A"]);
17765        g(
17766            &repo,
17767            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17768        );
17769        g(&repo, &["checkout", "-q", "main"]);
17770
17771        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17772        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17773        // Without a readable tracking ref the recorded base's merge base is used.
17774        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17775        assert_eq!(fallback, c1);
17776
17777        let mut state = review_state(&[
17778            "chore(deps): update a crate",
17779            "fix(daemon): apply a chosen action",
17780        ]);
17781        state.repo = repo.clone();
17782        state.base_branch = "main".to_owned();
17783        state.base_commit = c1;
17784        refresh_reviewed_commits(&mut state, "feat/own").await;
17785        assert_eq!(
17786            state.reviewed_commits,
17787            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17788        );
17789        assert_eq!(
17790            review_title(&state).as_deref(),
17791            Some("fix(daemon): apply a chosen action")
17792        );
17793        assert_eq!(
17794            leaked_subjects(&state, "feat/own").await,
17795            Some(vec!["chore(deps): update a crate".to_owned()])
17796        );
17797        // A stale tracking ref is still used when the fetch fails, but the
17798        // leak list is withheld.
17799        state.config.merge.remote = "nowhere".to_owned();
17800        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17801    }
17802
17803    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
17804    /// with `origin/main` one commit behind it, and a summarizing agent that
17805    /// prints whatever `reply` holds.
17806    struct SummaryFixture {
17807        _tmp: tempfile::TempDir,
17808        repo: PathBuf,
17809        reply: PathBuf,
17810        runner: Runner,
17811    }
17812
17813    fn summary_fixture() -> SummaryFixture {
17814        ask_test_home();
17815        let tmp = tempfile::tempdir().unwrap();
17816        let repo = tmp.path().join("repo");
17817        std::fs::create_dir_all(&repo).unwrap();
17818        init_repo(&repo);
17819        let origin = tmp.path().join("origin.git");
17820        let g = |args: &[&str]| {
17821            let out = std::process::Command::new("git")
17822                .args(args)
17823                .current_dir(&repo)
17824                .quiet()
17825                .output()
17826                .expect("spawn git");
17827            assert!(
17828                out.status.success(),
17829                "git {args:?}: {}",
17830                String::from_utf8_lossy(&out.stderr)
17831            );
17832        };
17833        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17834        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17835        g(&["push", "-q", "origin", "main"]);
17836        g(&["checkout", "-q", "-b", "feat/own"]);
17837        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17838        g(&["add", "-A"]);
17839        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17840        let reply = tmp.path().join("reply.txt");
17841        std::fs::write(
17842            &reply,
17843            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17844        )
17845        .unwrap();
17846        let mut runner = runner_at(RunStatus::Gating);
17847        let mut state = review_state(&["feat(own): add the own file"]);
17848        state.repo = repo.clone();
17849        state.base_branch = "main".to_owned();
17850        state.candidates[0].branch = "feat/own".to_owned();
17851        state.candidates[0].worktree = repo.clone();
17852        let mut author = spec("alpha");
17853        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17854        author
17855            .env
17856            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17857        state.config.agents = vec![author];
17858        runner.state = state;
17859        SummaryFixture {
17860            _tmp: tmp,
17861            repo,
17862            reply,
17863            runner,
17864        }
17865    }
17866
17867    async fn posted(f: &mut SummaryFixture) -> PrMessage {
17868        let winner = f.runner.state.candidates[0].clone();
17869        f.runner
17870            .guarded_pr_message(&winner, None, true)
17871            .await
17872            .unwrap()
17873            .expect("an English description never parks")
17874    }
17875
17876    fn clean_round_with_minor_findings() -> ReviewRound {
17877        let mut round = passing_round(2, None);
17878        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17879        round.reviews = vec![ReviewRecord {
17880            attempts: 0,
17881            reviewer: 1,
17882            agent: "alpha".into(),
17883            summary: "内部レビュー".into(),
17884            findings: vec![
17885                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17886                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17887            ],
17888            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17889            failed: None,
17890            duration_ms: 0,
17891        }];
17892        round
17893    }
17894
17895    #[tokio::test]
17896    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17897        for review_only in [false, true] {
17898            let mut f = summary_fixture();
17899            if !review_only {
17900                f.runner.state.candidates[0].agent = "alpha".into();
17901            }
17902            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17903            assert!(f.runner.state.reviews[0].clean);
17904            assert!(f.runner.state.reviews[0].fix.is_none());
17905            assert!(f.runner.state.open_findings().is_empty());
17906            assert_eq!(
17907                crate::pr_outcome::required_finding_ids(&f.runner.state),
17908                ["R2-1-1", "R2-1-2"]
17909            );
17910            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17911            let evidence = crate::pr_outcome::summary_evidence(
17912                &f.runner.state,
17913                crate::pr_outcome::FinalHead::At(&head),
17914            );
17915            assert!(
17916                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17917            );
17918            // The ordinary reply omits both IDs: no cached description is adopted.
17919            let omitted = posted(&mut f).await;
17920            assert!(omitted.body.contains("No description"));
17921            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17922            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17923            assert!(f.runner.state.change_summary.is_none());
17924            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17925            let described = posted(&mut f).await;
17926            assert!(described.body.contains("improve documentation for callers"));
17927            assert!(described.body.contains("remaining edge case"));
17928            assert!(described.body.contains("## Open review findings"));
17929            assert!(!described.body.contains("未修正"));
17930        }
17931    }
17932
17933    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17934        let mut f = summary_fixture();
17935        if !review_only {
17936            f.runner.state.candidates[0].agent = "alpha".into();
17937        }
17938        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17939        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17940        fixed
17941            .fix
17942            .as_mut()
17943            .unwrap()
17944            .rejected
17945            .push(crate::verdict::Rejection {
17946                id: "R1-1-1".into(),
17947                why: "呼び出されないコードのため辞退".into(),
17948            });
17949        let mut clean = clean_round_with_minor_findings();
17950        clean.verified_head = Some(head.clone());
17951        f.runner.state.reviews = vec![fixed, clean];
17952        ok_gate(&mut f.runner.state, Some(&head));
17953        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17954        f.runner.state.config.agents.push(scripted("beta", reply));
17955        f.runner.state.config.roles.language_judge =
17956            Some(crate::config::AgentChoice::One("beta".into()));
17957        f
17958    }
17959
17960    #[tokio::test]
17961    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17962        for review_only in [false, true] {
17963            let mut f = rewrite_fixture_with_findings(review_only,
17964                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17965            let m = posted(&mut f).await;
17966            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17967            let expected =
17968                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17969            assert!(m.body.contains(&expected));
17970            assert!(m.body.contains("## Review rounds"));
17971            assert!(
17972                m.body
17973                    .contains("Gate (`verify.gate`): passed on the final commit")
17974            );
17975            assert!(
17976                m.body
17977                    .contains("Tests (`verify.e2e`): passed on the final commit")
17978            );
17979            let c = f.runner.state.change_summary.as_ref().unwrap();
17980            assert!(
17981                m.body
17982                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17983            );
17984            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17985            assert!(crate::pr_outcome::covers_required_findings(
17986                &f.runner.state,
17987                &m.body
17988            ));
17989            assert!(m.body.contains("documentation remains incomplete"));
17990            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17991            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17992            assert!(f.runner.state.github_text.is_none());
17993        }
17994    }
17995
17996    #[tokio::test]
17997    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17998        for review_only in [false, true] {
17999            for reply in [
18000                "TITLE: fix: public description\nA short English description.",
18001                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
18002            ] {
18003                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
18004                let winner = f.runner.state.candidates[0].clone();
18005                let posted = f
18006                    .runner
18007                    .guarded_pr_message(&winner, None, true)
18008                    .await
18009                    .unwrap();
18010                assert!(posted.is_none());
18011                assert!(f.runner.state.parked);
18012                let withheld = f.runner.state.github_text.as_ref().unwrap();
18013                assert!(
18014                    withheld.rewrite.is_none(),
18015                    "an incomplete rewrite must not be selectable"
18016                );
18017                let q = ask::Questions::open()
18018                    .get(withheld.question.as_ref().unwrap())
18019                    .unwrap();
18020                assert!(
18021                    !q.choices
18022                        .contains(&crate::github_text::USE_REWRITE.to_owned())
18023                );
18024                assert!(
18025                    q.detail.contains("R1-1-1")
18026                        && q.detail.contains("R2-1-1")
18027                        && q.detail.contains("R2-1-2")
18028                );
18029            }
18030        }
18031    }
18032
18033    #[tokio::test]
18034    async fn japanese_reports_stay_local_in_both_public_run_modes() {
18035        for review_only in [false, true] {
18036            let mut f = summary_fixture();
18037            if !review_only {
18038                f.runner.state.candidates[0].agent = "alpha".into();
18039            }
18040            f.runner.state.instruction = "内部タスクの説明".into();
18041            f.runner.state.candidates[0].summary = "内部の候補報告".into();
18042            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18043            f.runner.state.reviews = vec![
18044                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18045                round_with_notes(2, Some("二回目の内部修正報告です。")),
18046                passing_round(3, Some(&head)),
18047            ];
18048            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18049            ok_gate(&mut f.runner.state, Some(&head));
18050            let captured = f.repo.join("public-prompt.txt");
18051            let author = &mut f.runner.state.config.agents[0];
18052            author.command = vec![
18053                "sh".into(),
18054                "-c".into(),
18055                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18056            ];
18057            author.env.insert(
18058                "PROMPT_FILE".into(),
18059                captured.to_string_lossy().into_owned(),
18060            );
18061            let m = posted(&mut f).await;
18062            assert!(m.body.contains("Adds `own.txt`"));
18063            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18064            assert!(m.body.contains("R1-1-1"));
18065            assert!(
18066                m.body
18067                    .contains("Gate (`verify.gate`): passed on the final commit")
18068            );
18069            assert!(
18070                m.body
18071                    .contains("Tests (`verify.e2e`): passed on the final commit")
18072            );
18073            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18074            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18075            let prompt = std::fs::read_to_string(captured).unwrap();
18076            assert!(prompt.contains(&format!(
18077                "git diff {} {head}",
18078                f.runner.state.change_summary.as_ref().unwrap().base
18079            )));
18080            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18081            assert_eq!(
18082                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18083                "二回目の内部修正報告です。"
18084            );
18085        }
18086    }
18087
18088    #[tokio::test]
18089    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18090        for review_only in [false, true] {
18091            let mut f = summary_fixture();
18092            if !review_only {
18093                f.runner.state.candidates[0].agent = "alpha".into();
18094            }
18095            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18096            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18097            std::fs::write(&f.reply, "no usable description").unwrap();
18098            let failed = posted(&mut f).await;
18099            assert!(
18100                failed
18101                    .body
18102                    .contains("No description of what this change does could be generated")
18103            );
18104            assert!(failed.body.is_ascii());
18105            assert!(f.runner.state.github_text.is_none());
18106            assert!(f.runner.state.change_summary.is_none());
18107            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18108            let evidence = crate::pr_outcome::summary_evidence(
18109                &f.runner.state,
18110                crate::pr_outcome::FinalHead::At(&head),
18111            );
18112            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18113            assert!(
18114                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18115                    .exists()
18116            );
18117            f.runner.state =
18118                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18119            std::fs::write(
18120                &f.reply,
18121                "TITLE: feat(own): add the own file\nAdds the committed own file.",
18122            )
18123            .unwrap();
18124            let retried = posted(&mut f).await;
18125            assert!(retried.body.contains("Adds the committed own file."));
18126            assert_eq!(
18127                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18128                "日本語の修正報告"
18129            );
18130        }
18131    }
18132
18133    #[tokio::test]
18134    async fn same_head_verification_and_findings_invalidate_public_prose() {
18135        let mut f = summary_fixture();
18136        posted(&mut f).await;
18137        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18138        let original = f
18139            .runner
18140            .state
18141            .change_summary
18142            .as_ref()
18143            .unwrap()
18144            .evidence
18145            .clone();
18146        ok_gate(&mut f.runner.state, Some(&head));
18147        std::fs::write(
18148            &f.reply,
18149            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18150        )
18151        .unwrap();
18152        let verified = posted(&mut f).await;
18153        assert!(verified.body.contains("now verified"));
18154        assert_ne!(
18155            f.runner.state.change_summary.as_ref().unwrap().evidence,
18156            original
18157        );
18158        let mut round = round_with_notes(1, Some("内部報告"));
18159        round.reviews.push(ReviewRecord {
18160            attempts: 0,
18161            reviewer: 1,
18162            agent: "alpha".into(),
18163            summary: "内部レビュー".into(),
18164            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18165            vote: None,
18166            failed: None,
18167            duration_ms: 0,
18168        });
18169        round
18170            .fix
18171            .as_mut()
18172            .unwrap()
18173            .rejected
18174            .push(crate::verdict::Rejection {
18175                id: "R1-1-1".into(),
18176                why: "呼び出し元から到達できない".into(),
18177            });
18178        f.runner.state.reviews = vec![round];
18179        // A description omitting the required ID is unusable, even with a title.
18180        let omitted = posted(&mut f).await;
18181        assert!(omitted.body.contains("No description"));
18182        assert!(omitted.body.contains("R1-1-1"));
18183        assert!(!omitted.body.contains("到達"));
18184        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18185        let described = posted(&mut f).await;
18186        assert!(described.body.contains("unreachable from callers"));
18187        assert!(described.body.contains("R1-1-1"));
18188        assert!(!described.body.contains("到達"));
18189        let evidence = f
18190            .runner
18191            .state
18192            .change_summary
18193            .as_ref()
18194            .unwrap()
18195            .evidence
18196            .clone();
18197        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18198        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18199        assert!(posted(&mut f).await.body.contains("rationale updated"));
18200        assert_ne!(
18201            f.runner.state.change_summary.as_ref().unwrap().evidence,
18202            evidence
18203        );
18204        // Old serialized summaries have no evidence fingerprint and must regenerate.
18205        f.runner
18206            .state
18207            .change_summary
18208            .as_mut()
18209            .unwrap()
18210            .evidence
18211            .clear();
18212        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18213        assert!(posted(&mut f).await.body.contains("current specification"));
18214    }
18215
18216    #[tokio::test]
18217    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18218        let mut f = summary_fixture();
18219        let m = posted(&mut f).await;
18220        assert_eq!(m.title, "feat(own): add the own file");
18221        assert!(
18222            m.body
18223                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
18224            "{}",
18225            m.body
18226        );
18227        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
18228        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
18229        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18230        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18231        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
18232        let kept = f.runner.state.change_summary.as_ref().unwrap();
18233        assert_eq!(
18234            (kept.head.as_str(), kept.base.as_str()),
18235            (head.as_str(), base.as_str())
18236        );
18237        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18238    }
18239
18240    #[tokio::test]
18241    async fn review_only_summary_receives_the_configured_github_language() {
18242        let mut f = summary_fixture();
18243        f.runner.state.config.graph.github_language = "ja".into();
18244        let captured = f.repo.join("summary-prompt.txt");
18245        let author = &mut f.runner.state.config.agents[0];
18246        author.command = vec![
18247            "sh".into(),
18248            "-c".into(),
18249            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18250        ];
18251        author.env.insert(
18252            "PROMPT_FILE".into(),
18253            captured.to_string_lossy().into_owned(),
18254        );
18255        std::fs::write(
18256            &f.reply,
18257            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18258        )
18259        .unwrap();
18260        let m = posted(&mut f).await;
18261        let prompt = std::fs::read_to_string(captured).unwrap();
18262        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18263        assert!(!prompt.contains("in English"));
18264        assert_eq!(m.title, "feat(own): ファイルを追加");
18265        assert!(m.body.contains("own.txt を追加します。"));
18266    }
18267
18268    #[tokio::test]
18269    async fn review_fixes_come_after_the_whole_change_summary() {
18270        let mut f = summary_fixture();
18271        f.runner.state.reviews = vec![
18272            round_with_notes(1, Some("handled the PATH case")),
18273            round_with_notes(2, None),
18274        ];
18275        let m = posted(&mut f).await;
18276        let summary = m.body.find("## Summary").expect("summary");
18277        let fixes = m.body.find("## Review rounds").expect("review rounds");
18278        assert!(summary < fixes, "{}", m.body);
18279        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18280        assert!(m.body.contains("## Final verification"), "{}", m.body);
18281    }
18282
18283    #[tokio::test]
18284    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18285        let mut f = summary_fixture();
18286        let first = posted(&mut f).await;
18287        let first_head = f.runner.state.change_summary.clone().unwrap().head;
18288        // The same head reuses the record: a different reply is not read.
18289        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18290        let again = posted(&mut f).await;
18291        assert_eq!((again.title, again.body), (first.title, first.body));
18292
18293        let g = |args: &[&str]| {
18294            let out = std::process::Command::new("git")
18295                .args(args)
18296                .current_dir(&f.repo)
18297                .quiet()
18298                .output()
18299                .expect("spawn git");
18300            assert!(out.status.success(), "git {args:?}");
18301        };
18302        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18303        g(&["add", "-A"]);
18304        g(&["commit", "-q", "-m", "fix(own): handle more"]);
18305        let second = posted(&mut f).await;
18306        let kept = f.runner.state.change_summary.clone().unwrap();
18307        assert_ne!(kept.head, first_head);
18308        assert!(second.body.contains("Not read yet."), "{}", second.body);
18309        assert!(!second.body.contains("so the first summary is read"));
18310        assert_eq!(second.title, "feat(own): other");
18311    }
18312
18313    #[tokio::test]
18314    async fn an_unavailable_summary_is_stated_and_never_invented() {
18315        let mut f = summary_fixture();
18316        std::fs::write(&f.reply, "no title line here").unwrap();
18317        let m = posted(&mut f).await;
18318        assert!(
18319            m.body
18320                .contains("No description of what this change does could be generated"),
18321            "{}",
18322            m.body
18323        );
18324        assert!(m.body.contains("internal reports remain in local records"));
18325        assert!(f.runner.state.change_summary.is_none());
18326        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18327        // A preview asks nobody and claims nothing.
18328        let winner = f.runner.state.candidates[0].clone();
18329        let preview = f
18330            .runner
18331            .guarded_pr_message(&winner, None, false)
18332            .await
18333            .unwrap()
18334            .unwrap();
18335        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18336    }
18337
18338    #[test]
18339    fn pr_message_review_single_commit_uses_its_subject() {
18340        let state = review_state(&["feat(nats): per-role user"]);
18341        let m = pr_message(&state, 'A');
18342        assert_eq!(m.title, "feat(nats): per-role user");
18343        assert!(!m.body.contains("Review the work already"), "{}", m.body);
18344        assert!(m.body.contains("## Commits under review"), "{}", m.body);
18345    }
18346
18347    #[test]
18348    fn pr_message_review_multi_commit_takes_the_oldest() {
18349        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18350        let m = pr_message(&state, 'A');
18351        assert_eq!(m.title, "feat: the change");
18352        for s in ["feat: the change", "fix: typo", "fix: again"] {
18353            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18354        }
18355    }
18356
18357    #[test]
18358    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18359        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18360            let mut state = review_state(&[first, "fix: later fixup"]);
18361            state.candidates[0].branch = "機能/ログイン".to_owned();
18362            let m = pr_message(&state, 'A');
18363            assert!(
18364                m.title.starts_with("chore: land candidate A of run"),
18365                "{}",
18366                m.title
18367            );
18368        }
18369    }
18370
18371    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18372        BranchFacts {
18373            commits: commits
18374                .iter()
18375                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18376                .collect(),
18377            stat: stat.to_owned(),
18378        }
18379    }
18380
18381    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18382        let mut r = review_round(true, 0, 1, 1, true, true);
18383        r.round = round;
18384        r.fix = notes.map(|n| FixRecord {
18385            agent: "fixer".to_owned(),
18386            addressed: Vec::new(),
18387            rejected: Vec::new(),
18388            notes: n.to_owned(),
18389            committed: true,
18390            failed: None,
18391            duration_ms: 0,
18392            continuation: None,
18393        });
18394        r
18395    }
18396
18397    #[test]
18398    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18399        let state = review_state(&["ignored"]);
18400        let f = facts(
18401            &[
18402                (
18403                    "fix(login): resolve PATH on macOS",
18404                    "Login shells skip rc files.",
18405                ),
18406                ("fix: address review", ""),
18407            ],
18408            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18409        );
18410        let m = pr_message_with(&state, 'A', Some(&f));
18411        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18412        assert!(
18413            m.body
18414                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
18415            "{}",
18416            m.body
18417        );
18418        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18419        assert!(m.body.contains("## Diff stat"), "{}", m.body);
18420        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18421        for banned in [
18422            "Review the work already",
18423            "no task statement",
18424            "Original task",
18425        ] {
18426            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18427        }
18428        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18429    }
18430
18431    #[test]
18432    fn pr_message_review_truncates_a_huge_first_commit_body() {
18433        let state = review_state(&["ignored"]);
18434        let f = facts(
18435            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18436            "s",
18437        );
18438        let m = pr_message_with(&state, 'A', Some(&f));
18439        assert!(m.body.len() < 30_000, "{}", m.body.len());
18440        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18441        assert!(
18442            m.body.contains("- ... 1 more commit(s)"),
18443            "{}",
18444            m.body.len()
18445        );
18446        assert!(m.body.ends_with("magi:candidate-a\n"));
18447    }
18448
18449    #[test]
18450    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18451        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18452        assert_eq!(m.title, "feat: x");
18453        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18454        assert!(!m.body.contains("Diff stat"), "{}", m.body);
18455        assert!(!m.body.contains("no task statement"), "{}", m.body);
18456    }
18457
18458    #[test]
18459    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18460        let mut state = review_state(&["日本語の件名"]);
18461        state.candidates[0].branch = "fix/macos-login-path".to_owned();
18462        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18463        state.candidates[0].branch = "機能/ログイン".to_owned();
18464        assert!(
18465            pr_message(&state, 'A')
18466                .title
18467                .starts_with("chore: land candidate A")
18468        );
18469    }
18470
18471    #[test]
18472    fn pr_message_review_fixes_survive_a_clean_final_round() {
18473        let mut state = review_state(&["feat: x"]);
18474        state.reviews = vec![
18475            round_with_notes(1, Some("handled the PATH case")),
18476            round_with_notes(2, None),
18477        ];
18478        let body = pr_message(&state, 'A').body;
18479        assert!(
18480            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18481            "{body}"
18482        );
18483        assert!(body.contains("### Round 1"), "{body}");
18484
18485        state.reviews = vec![
18486            round_with_notes(1, Some("first fix")),
18487            round_with_notes(2, Some("")),
18488            round_with_notes(3, Some("second fix")),
18489            round_with_notes(4, None),
18490        ];
18491        let body = pr_message(&state, 'A').body;
18492        assert!(
18493            body.contains("### Round 1") && !body.contains("first fix"),
18494            "{body}"
18495        );
18496        assert!(
18497            body.contains("### Round 3") && !body.contains("second fix"),
18498            "{body}"
18499        );
18500        assert!(body.contains("### Round 2"), "{body}");
18501    }
18502
18503    const FINAL: &str = "abcdef1234567890";
18504
18505    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18506        let mut r = round_with_notes(round, None);
18507        r.verified_head = verified.map(str::to_owned);
18508        r
18509    }
18510
18511    fn ok_gate(state: &mut RunState, head: Option<&str>) {
18512        state.gate = vec![CommandOutcome {
18513            command: "gate".to_owned(),
18514            code: Some(0),
18515            output_tail: String::new(),
18516            duration_ms: 0,
18517            resource_blocked: false,
18518        }];
18519        state.gate_ran = true;
18520        state.gate_head = head.map(str::to_owned);
18521    }
18522
18523    fn body_at(state: &RunState) -> String {
18524        pr_message_raw(
18525            state,
18526            'A',
18527            None,
18528            crate::pr_outcome::FinalHead::At(FINAL),
18529            ChangeNote::NotAsked,
18530        )
18531        .body
18532    }
18533
18534    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18535
18536    #[test]
18537    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18538        let mut state = review_state(&["feat: x"]);
18539        state.reviews = vec![
18540            round_with_notes(1, Some(SANDBOXED)),
18541            passing_round(2, Some(FINAL)),
18542        ];
18543        ok_gate(&mut state, Some(FINAL));
18544        let body = body_at(&state);
18545        assert!(!body.contains("Handled the PATH case."), "{body}");
18546        assert!(!body.contains("could not commit"), "{body}");
18547        assert!(!body.contains("could not run"), "{body}");
18548        assert!(
18549            body.contains("The fix is committed on the branch"),
18550            "{body}"
18551        );
18552        assert!(!body.contains("sandbox"), "{body}");
18553        assert!(
18554            body.contains("Tests (`verify.e2e`): passed on the final commit"),
18555            "{body}"
18556        );
18557        assert!(
18558            body.contains("Gate (`verify.gate`): passed on the final commit"),
18559            "{body}"
18560        );
18561        // The raw report is untouched in the record.
18562        assert!(
18563            state.reviews[0]
18564                .fix
18565                .as_ref()
18566                .unwrap()
18567                .notes
18568                .contains("could not commit")
18569        );
18570    }
18571
18572    #[test]
18573    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18574        let mut state = review_state(&["feat: x"]);
18575        let mut r1 = round_with_notes(1, Some(SANDBOXED));
18576        r1.fix.as_mut().unwrap().committed = false;
18577        let mut r2 = passing_round(2, Some(FINAL));
18578        r2.e2e[0].code = Some(1);
18579        state.reviews = vec![r1, r2];
18580        let body = body_at(&state);
18581        assert!(!body.contains("could not commit"), "{body}");
18582        assert!(!body.contains("could not run"), "{body}");
18583        assert!(body.contains("This round produced no commit"), "{body}");
18584        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18585        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18586        assert!(!body.contains("passed"), "{body}");
18587    }
18588
18589    #[test]
18590    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18591        let mut state = review_state(&["feat: x"]);
18592        state.reviews = vec![
18593            round_with_notes(1, Some(SANDBOXED)),
18594            passing_round(2, Some("0123456789abcdef")),
18595        ];
18596        ok_gate(&mut state, Some("0123456789abcdef"));
18597        let body = body_at(&state);
18598        assert!(!body.contains("HTTP tests could not run"), "{body}");
18599        // The commit claim is still settled by the structured record.
18600        assert!(!body.contains("could not commit"), "{body}");
18601        assert!(
18602            body.contains("passed on an earlier commit (`0123456`)"),
18603            "{body}"
18604        );
18605        assert!(!body.contains("passed on the final commit"), "{body}");
18606
18607        // A gate recorded before `gate_head` existed is "not recorded".
18608        ok_gate(&mut state, None);
18609        let body = body_at(&state);
18610        assert!(
18611            body.contains("passed on a commit that was not recorded"),
18612            "{body}"
18613        );
18614    }
18615
18616    #[test]
18617    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18618        let state = state_with_summary(
18619            "add retries to the client",
18620            "TITLE: feat: retries\n\nDid it.",
18621        );
18622        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18623        assert_eq!(m.title, "feat: retries");
18624        assert!(
18625            m.body.contains("<summary>Original task</summary>"),
18626            "{}",
18627            m.body
18628        );
18629        assert!(!m.body.contains("Commits under review"), "{}", m.body);
18630        assert!(
18631            m.body
18632                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18633            "{}",
18634            m.body
18635        );
18636    }
18637
18638    #[test]
18639    fn pr_message_review_bounds_a_long_english_subject() {
18640        let long = format!("feat: {}", "word ".repeat(100));
18641        let m = pr_message(&review_state(&[&long]), 'A');
18642        assert!(m.title.starts_with("feat: word"), "{}", m.title);
18643        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18644    }
18645
18646    #[test]
18647    fn pr_message_implementation_run_is_unchanged_by_review_support() {
18648        let state = state_with_summary("add retries\n\ndetails", "- did some things");
18649        let m = pr_message(&state, 'A');
18650        assert_eq!(m.title, "add retries");
18651        assert!(m.body.contains("<summary>Original task</summary>"));
18652        assert!(!m.body.contains("Commits under review"));
18653        assert_eq!(landing_subject_source(&state), state.instruction);
18654    }
18655
18656    #[test]
18657    fn review_run_squash_subject_is_the_change_not_the_prompt() {
18658        let state = review_state(&["feat: the change", "fix: typo"]);
18659        let source = landing_subject_source(&state);
18660        assert_eq!(land::merge_subject("", &source), "feat: the change");
18661        assert_eq!(
18662            land::merge_subject("magi: candidate A (uncommitted work)", &source),
18663            "feat: the change"
18664        );
18665        // An operator's rename still wins.
18666        assert_eq!(
18667            land::merge_subject("feat: renamed by hand", &source),
18668            "feat: renamed by hand"
18669        );
18670        let blank = review_state(&["日本語"]);
18671        assert!(
18672            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18673        );
18674    }
18675
18676    #[test]
18677    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18678        let state = review_state(&["feat: the change"]);
18679        let source = landing_subject_source(&state);
18680        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18681        assert_eq!(
18682            land::merge_subject(landing_title(&state, old), &source),
18683            "feat: the change"
18684        );
18685        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18686        let task = state_with_summary("add retries", "");
18687        assert_eq!(landing_title(&task, old), old);
18688    }
18689
18690    #[test]
18691    fn pr_message_describes_the_change_not_the_task() {
18692        let state = state_with_summary(
18693            "今回やってほしいこと: results projector を直す",
18694            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18695        );
18696        let m = pr_message(&state, 'A');
18697        assert_eq!(m.title, "fix(web): batch the runs list reads");
18698        assert!(
18699            m.body.starts_with("## Summary\n\n- reads run.json once"),
18700            "{}",
18701            m.body
18702        );
18703        assert!(!m.body.contains("TITLE:"), "{}", m.body);
18704        let task_at = m.body.find("今回やってほしいこと").unwrap();
18705        let details_at = m.body.find("<details>").unwrap();
18706        assert!(
18707            details_at < task_at,
18708            "the task lives inside <details>: {}",
18709            m.body
18710        );
18711        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18712        assert!(m.body.contains("magi:candidate-a"));
18713    }
18714
18715    #[test]
18716    fn pr_message_falls_back_to_the_task_without_a_title_line() {
18717        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18718        let m = pr_message(&state, 'A');
18719        assert_eq!(m.title, "add retries");
18720        assert!(
18721            m.body.contains("## Summary\n\n- did some things"),
18722            "{}",
18723            m.body
18724        );
18725
18726        let none = RunState::new(
18727            PathBuf::from("/repo"),
18728            "main".to_owned(),
18729            "abc1234".to_owned(),
18730            "add retries".to_owned(),
18731            Config::default(),
18732        );
18733        let m = pr_message(&none, 'A');
18734        assert_eq!(m.title, "add retries");
18735        assert!(!m.body.contains("## Summary"), "{}", m.body);
18736    }
18737
18738    #[test]
18739    fn pr_message_refuses_the_candidate_commit_subject() {
18740        for bad in [
18741            "TITLE: magi: candidate A (uncommitted work)",
18742            "TITLE: chore: stuff (uncommitted work)",
18743            "TITLE:   ",
18744        ] {
18745            let state = state_with_summary("add retries", bad);
18746            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18747        }
18748    }
18749
18750    #[test]
18751    fn pr_message_bounds_a_very_long_task_and_title() {
18752        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18753        let state = state_with_summary(&long, "- nothing");
18754        let m = pr_message(&state, 'A');
18755        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18756        assert!(!m.title.contains('\n'));
18757
18758        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18759        let m = pr_message(&state, 'A');
18760        assert!(m.title.starts_with("feat: "));
18761        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18762        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18763    }
18764
18765    fn long_title_of(instruction: &str) -> String {
18766        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18767    }
18768
18769    #[test]
18770    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18771        let first = "Make the landing path keep a readable title for long tasks";
18772        let line = format!(
18773            "{first}. {}",
18774            "And then keep going with more words ".repeat(20)
18775        );
18776        let t = long_title_of(&line);
18777        assert_eq!(t, first);
18778        assert!(!t.starts_with("chore: land"));
18779    }
18780
18781    #[test]
18782    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18783        let line = "word ".repeat(200);
18784        let t = long_title_of(&line);
18785        assert!(t.ends_with("word..."), "{t}");
18786        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18787    }
18788
18789    #[test]
18790    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18791        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18792            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18793        }
18794    }
18795
18796    #[test]
18797    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18798        let mut state = state_with_summary("再試行を追加する", "- nothing");
18799        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18800        state.config.graph.github_language = "ja".into();
18801        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18802    }
18803
18804    #[test]
18805    fn pr_message_title_limit_is_exact() {
18806        let at = "a".repeat(PR_TITLE_MAX);
18807        assert_eq!(long_title_of(&at), at);
18808        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18809        assert!(over.ends_with("..."), "{over}");
18810        assert_eq!(over.chars().count(), PR_TITLE_MAX);
18811    }
18812
18813    #[test]
18814    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18815        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18816        let t = long_title_of(&line);
18817        assert!(t.ends_with("..."), "{t}");
18818        assert!(t.is_ascii(), "{t}");
18819    }
18820
18821    #[test]
18822    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18823        let line = format!(
18824            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18825            "plus more filler words ".repeat(20)
18826        );
18827        let t = long_title_of(&line);
18828        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18829    }
18830
18831    #[test]
18832    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18833        let line = format!("\n\n# {}", "title words ".repeat(40));
18834        let state = state_with_summary(&line, "- nothing");
18835        let m = pr_message(&state, 'A');
18836        assert!(m.title.starts_with("title words"), "{}", m.title);
18837        assert_eq!(
18838            land::merge_subject(&m.title, &landing_subject_source(&state)),
18839            m.title
18840        );
18841        // An operator's rename wins untouched.
18842        assert_eq!(
18843            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18844            "feat: renamed by hand"
18845        );
18846    }
18847
18848    #[tokio::test]
18849    async fn github_text_rewrite_is_bounded_and_falls_back() {
18850        crate::run::pin_test_home();
18851        for (reply, accepted) in [
18852            (
18853                "TITLE: fix: retries\nAdd retries for failed requests.",
18854                true,
18855            ),
18856            (
18857                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18858                false,
18859            ),
18860            (
18861                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18862                true,
18863            ),
18864            (
18865                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18866                true,
18867            ),
18868            (
18869                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18870                true,
18871            ),
18872            (
18873                "TITLE: fix: retries\n日本語の説明です。 token=secret",
18874                false,
18875            ),
18876            ("TITLE: fix: retries\n", false),
18877        ] {
18878            let dir = tempfile::tempdir().unwrap();
18879            let mut runner = runner_at(RunStatus::Gating);
18880            runner.state = state_with_summary(
18881                "add retries",
18882                "TITLE: fix: retries\n日本語の説明を書きます。",
18883            );
18884            runner.state.repo = dir.path().to_owned();
18885            runner.state.candidates[0].worktree = dir.path().to_owned();
18886            let mut author = spec("alpha");
18887            author.command = vec![
18888                "sh".into(),
18889                "-c".into(),
18890                "printf '%s' \"$REWRITE_REPLY\"".into(),
18891            ];
18892            author.env.insert("REWRITE_REPLY".into(), reply.into());
18893            runner.state.config.agents = vec![author];
18894            seed_public_summary(&mut runner);
18895            let winner = runner.state.candidates[0].clone();
18896            let first = runner
18897                .guarded_pr_message(&winner, None, true)
18898                .await
18899                .unwrap();
18900            let message = if accepted {
18901                first.expect("a passing rewrite never asks")
18902            } else {
18903                // Withheld: the run parks on a question and posts nothing yet.
18904                assert!(first.is_none());
18905                assert!(runner.state.parked);
18906                let store = ask::Questions::open();
18907                let asked: Vec<_> = store
18908                    .list()
18909                    .into_iter()
18910                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18911                    .collect();
18912                assert_eq!(asked.len(), 1);
18913                assert!(!asked[0].detail.contains("secret"));
18914                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18915                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18916                assert_eq!(
18917                    asked[0]
18918                        .choices
18919                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18920                    rewrite.is_some()
18921                );
18922                // Silence: the question lapses and the neutral text is used,
18923                // without a second rewrite call or a second question.
18924                store
18925                    .update(&asked[0].id, |q| {
18926                        q.abandon("test");
18927                        Ok(())
18928                    })
18929                    .unwrap();
18930                let again = runner
18931                    .guarded_pr_message(&winner, None, true)
18932                    .await
18933                    .unwrap()
18934                    .expect("an abandoned question falls back");
18935                let n = store
18936                    .list()
18937                    .into_iter()
18938                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18939                    .count();
18940                assert_eq!(n, 1, "asked once per rejected text");
18941                again
18942            };
18943            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18944            assert!(!message.body.contains("secret"));
18945            assert!(!message.body.contains("prefix"));
18946            assert!(!message.title.contains("secret"));
18947            assert_eq!(
18948                message.body.contains("Add retries for failed requests."),
18949                accepted
18950            );
18951            assert_eq!(
18952                runner.state.seats["github-text-rewrite"].turns, 1,
18953                "only one rewrite invocation"
18954            );
18955            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18956            if accepted {
18957                assert!(runner.state.github_text.is_none());
18958                assert!(!runner.state.parked);
18959                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18960                assert!(message.body.contains("## Final verification"));
18961                assert!(message.body.contains("Evidence: final diff"));
18962                assert!(
18963                    message
18964                        .body
18965                        .contains(&format!("magi:run/{}", runner.state.id))
18966                );
18967            }
18968            if !accepted {
18969                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18970                assert!(
18971                    message
18972                        .body
18973                        .contains(&format!("magi:run/{}", runner.state.id))
18974                );
18975            }
18976        }
18977    }
18978
18979    fn scripted(id: &str, reply: &str) -> AgentSpec {
18980        let mut a = spec(id);
18981        a.command = vec![
18982            "sh".into(),
18983            "-c".into(),
18984            "printf '%s' \"$REWRITE_REPLY\"".into(),
18985        ];
18986        a.env.insert("REWRITE_REPLY".into(), reply.into());
18987        a
18988    }
18989
18990    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18991    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18992
18993    /// Gate tests inject generated public output, not internal reports.
18994    fn seed_public_summary(runner: &mut Runner) {
18995        let repo = &runner.state.repo;
18996        let g = |args: &[&str]| {
18997            let out = std::process::Command::new("git")
18998                .args(args)
18999                .current_dir(repo)
19000                .quiet()
19001                .output()
19002                .unwrap();
19003            assert!(out.status.success(), "git {args:?}");
19004            String::from_utf8_lossy(&out.stdout).trim().to_owned()
19005        };
19006        g(&["init", "-q", "-b", "main"]);
19007        g(&[
19008            "-c",
19009            "user.name=Test",
19010            "-c",
19011            "user.email=test@example.invalid",
19012            "commit",
19013            "-q",
19014            "--allow-empty",
19015            "-m",
19016            "feat: fixture",
19017        ]);
19018        let head = g(&["rev-parse", "HEAD"]);
19019        g(&["update-ref", "refs/remotes/origin/main", &head]);
19020        let evidence = crate::pr_outcome::summary_evidence(
19021            &runner.state,
19022            crate::pr_outcome::FinalHead::At(&head),
19023        );
19024        let generated = &runner.state.candidates[0].summary;
19025        runner.state.change_summary = Some(crate::run::ChangeSummary {
19026            base: head.clone(),
19027            head,
19028            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
19029            title: summary_title(generated).unwrap(),
19030            body: summary_without_title(generated),
19031        });
19032    }
19033
19034    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19035        let mut runner = runner_at(RunStatus::Gating);
19036        runner.state = state_with_summary("add retries", JA_SUMMARY);
19037        if review_only {
19038            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19039        }
19040        runner.state.repo = dir.to_owned();
19041        runner.state.candidates[0].worktree = dir.to_owned();
19042        runner.state.config.agents = agents;
19043        seed_public_summary(&mut runner);
19044        runner
19045    }
19046
19047    fn gate_questions(runner: &Runner) -> usize {
19048        ask::Questions::open()
19049            .list()
19050            .into_iter()
19051            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19052            .count()
19053    }
19054
19055    #[tokio::test]
19056    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19057        crate::run::pin_test_home();
19058        let dir = tempfile::tempdir().unwrap();
19059        let mut runner = rewrite_runner(
19060            dir.path(),
19061            true,
19062            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19063        );
19064        runner.state.config.roles.language_judge =
19065            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19066        let winner = runner.state.candidates[0].clone();
19067        let msg = runner
19068            .guarded_pr_message(&winner, None, true)
19069            .await
19070            .unwrap()
19071            .expect("an English rewrite posts without asking");
19072        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19073        assert!(msg.body.contains("<details>kept</details>"));
19074        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19075        assert_eq!(gate_questions(&runner), 0);
19076        assert!(runner.state.github_text.is_none());
19077        assert!(!runner.state.seats.contains_key("impl-A"));
19078        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19079    }
19080
19081    #[tokio::test]
19082    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19083        crate::run::pin_test_home();
19084        let dir = tempfile::tempdir().unwrap();
19085        let mut runner = rewrite_runner(
19086            dir.path(),
19087            false,
19088            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19089        );
19090        runner.state.config.roles.language_judge =
19091            Some(crate::config::AgentChoice::One("beta".into()));
19092        runner.state.seats.insert(
19093            "impl-A".into(),
19094            SeatState::new("impl-A", "alpha", runner.state.seed),
19095        );
19096        let winner = runner.state.candidates[0].clone();
19097        let msg = runner
19098            .guarded_pr_message(&winner, None, true)
19099            .await
19100            .unwrap()
19101            .expect("rewritten");
19102        assert!(msg.body.contains("<details>kept</details>"));
19103        assert_eq!(runner.state.seats["impl-A"].turns, 0);
19104        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19105        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19106    }
19107
19108    #[tokio::test]
19109    async fn rewrite_reuses_a_resumable_implementer_session() {
19110        crate::run::pin_test_home();
19111        let dir = tempfile::tempdir().unwrap();
19112        let mut runner = rewrite_runner(
19113            dir.path(),
19114            false,
19115            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19116        );
19117        runner.state.config.roles.language_judge =
19118            Some(crate::config::AgentChoice::One("beta".into()));
19119        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19120        seat.turns = 1;
19121        runner.state.seats.insert("impl-A".into(), seat);
19122        let winner = runner.state.candidates[0].clone();
19123        runner
19124            .guarded_pr_message(&winner, None, true)
19125            .await
19126            .unwrap()
19127            .expect("rewritten");
19128        assert_eq!(runner.state.seats["impl-A"].turns, 2);
19129        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19130    }
19131
19132    #[tokio::test]
19133    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19134        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
19135        // session, was out of quota, and nothing else was asked.
19136        crate::run::pin_test_home();
19137        let dir = tempfile::tempdir().unwrap();
19138        let mut exhausted = spec("alpha");
19139        exhausted.command = vec!["false".into()];
19140        let mut runner = rewrite_runner(
19141            dir.path(),
19142            false,
19143            vec![exhausted, scripted("beta", EN_REWRITE)],
19144        );
19145        // The implementer's own agent is in the role too: it is not asked twice.
19146        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19147            "alpha".into(),
19148            "beta".into(),
19149        ]));
19150        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19151        seat.turns = 1;
19152        runner.state.seats.insert("impl-A".into(), seat);
19153        let winner = runner.state.candidates[0].clone();
19154        let msg = runner
19155            .guarded_pr_message(&winner, None, true)
19156            .await
19157            .unwrap()
19158            .expect("the role rewrites after the implementer fails");
19159        assert!(msg.body.contains("`src/net.rs`"));
19160        assert_eq!(
19161            runner.state.seats["impl-A"].turns, 2,
19162            "the implementer was tried once"
19163        );
19164        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19165        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19166    }
19167
19168    #[tokio::test]
19169    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19170        crate::run::pin_test_home();
19171        let dir = tempfile::tempdir().unwrap();
19172        let mut broken = spec("bad");
19173        broken.command = vec!["false".into()];
19174        let mut runner = rewrite_runner(
19175            dir.path(),
19176            true,
19177            vec![
19178                broken,
19179                scripted("noformat", "Add retries."),
19180                scripted("good", EN_REWRITE),
19181            ],
19182        );
19183        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19184            "bad".into(),
19185            "noformat".into(),
19186            "good".into(),
19187        ]));
19188        let winner = runner.state.candidates[0].clone();
19189        let msg = runner
19190            .guarded_pr_message(&winner, None, true)
19191            .await
19192            .unwrap()
19193            .expect("the third agent rewrites");
19194        assert!(msg.body.contains("`src/net.rs`"));
19195        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19196        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19197    }
19198
19199    #[tokio::test]
19200    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19201        crate::run::pin_test_home();
19202        // (agents, role): still Japanese; no TITLE; nothing resolvable.
19203        for (agents, role, saved) in [
19204            (
19205                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19206                Some("beta"),
19207                true,
19208            ),
19209            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19210            (Vec::new(), None, false),
19211        ] {
19212            let dir = tempfile::tempdir().unwrap();
19213            let mut runner = rewrite_runner(dir.path(), true, agents);
19214            runner.state.config.roles.language_judge =
19215                role.map(|r| crate::config::AgentChoice::One(r.into()));
19216            let winner = runner.state.candidates[0].clone();
19217            let first = runner
19218                .guarded_pr_message(&winner, None, true)
19219                .await
19220                .unwrap();
19221            assert!(first.is_none(), "withheld text parks the run");
19222            assert!(runner.state.parked);
19223            assert_eq!(gate_questions(&runner), 1);
19224            let g = runner.state.github_text.as_ref().unwrap();
19225            assert_eq!(g.rewrite.is_some(), saved);
19226            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
19227        }
19228    }
19229
19230    #[tokio::test]
19231    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
19232        crate::run::pin_test_home();
19233        let dir = tempfile::tempdir().unwrap();
19234        let mut runner = runner_at(RunStatus::Gating);
19235        runner.state =
19236            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19237        runner.state.repo = dir.path().to_owned();
19238        runner.state.candidates[0].worktree = dir.path().to_owned();
19239        let mut author = spec("alpha");
19240        author.command = vec![
19241            "sh".into(),
19242            "-c".into(),
19243            "printf '%s' \"$REWRITE_REPLY\"".into(),
19244        ];
19245        author.env.insert(
19246            "REWRITE_REPLY".into(),
19247            format!(
19248                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19249                "English sentence. ".repeat(600)
19250            ),
19251        );
19252        runner.state.config.agents = vec![author];
19253        seed_public_summary(&mut runner);
19254        let winner = runner.state.candidates[0].clone();
19255        assert!(
19256            runner
19257                .guarded_pr_message(&winner, None, true)
19258                .await
19259                .unwrap()
19260                .is_none()
19261        );
19262        let record = runner.state.github_text.clone().unwrap();
19263        let rewrite = record.rewrite.clone().unwrap();
19264        assert_eq!(rewrite.failed, ["body-language"]);
19265        assert!(
19266            rewrite
19267                .body
19268                .ends_with(&format!("magi:run/{}", runner.state.id))
19269        );
19270        let store = ask::Questions::open();
19271        let q = store.get(record.question.as_ref().unwrap()).unwrap();
19272        assert!(
19273            q.choices
19274                .contains(&crate::github_text::USE_REWRITE.to_owned())
19275        );
19276        for label in [
19277            "ORIGINAL",
19278            "FALLBACK",
19279            "REWRITE",
19280            "Rules still failed: body-language",
19281            "truncated",
19282        ] {
19283            assert!(q.detail.contains(label), "{label}");
19284        }
19285        let artifact = std::fs::read_to_string(crate::run::artifact_path(
19286            &runner.state,
19287            &crate::github_text::artifact_name(&record.fingerprint),
19288        ))
19289        .unwrap();
19290        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19291        // Restart from serialized state, then retry after a bad owner title.
19292        runner.state =
19293            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19294        store
19295            .update(&q.id, |q| {
19296                q.say("まだ日本語")?;
19297                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19298            })
19299            .unwrap();
19300        assert!(
19301            runner
19302                .guarded_pr_message(&winner, None, true)
19303                .await
19304                .unwrap()
19305                .is_none()
19306        );
19307        let retry = runner.state.github_text.as_ref().unwrap();
19308        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19309        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19310        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19311        assert!(
19312            q.posting_comparison
19313                .as_ref()
19314                .unwrap()
19315                .matches_detail(&q.detail)
19316        );
19317        store
19318            .update(&q.id, |q| {
19319                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19320            })
19321            .unwrap();
19322        let message = runner
19323            .guarded_pr_message(&winner, None, true)
19324            .await
19325            .unwrap()
19326            .unwrap();
19327        assert_eq!(message.title, rewrite.title);
19328        assert_eq!(message.body, rewrite.body);
19329        assert_eq!(
19330            runner.state.github_text.as_ref().unwrap().choice,
19331            Some(GithubTextChoice::Rewrite)
19332        );
19333        let again = runner
19334            .guarded_pr_message(&winner, None, true)
19335            .await
19336            .unwrap()
19337            .unwrap();
19338        assert_eq!(again.title, message.title);
19339        assert_eq!(again.body, message.body);
19340        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19341        assert!(
19342            !runner
19343                .state
19344                .events
19345                .iter()
19346                .any(|e| e.message.contains(&rewrite.title)
19347                    || e.message.contains("English sentence."))
19348        );
19349    }
19350
19351    #[tokio::test]
19352    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19353        crate::run::pin_test_home();
19354        let dir = tempfile::tempdir().unwrap();
19355        let mut runner = runner_at(RunStatus::Gating);
19356        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19357        runner.state.repo = dir.path().to_owned();
19358        runner.state.candidates[0].worktree = dir.path().to_owned();
19359        seed_public_summary(&mut runner);
19360        let winner = runner.state.candidates[0].clone();
19361        assert!(
19362            runner
19363                .guarded_pr_message(&winner, None, true)
19364                .await
19365                .unwrap()
19366                .is_none()
19367        );
19368        let record = runner.state.github_text.as_ref().unwrap();
19369        assert!(record.rewrite.is_none());
19370        // Records written before the additive field deserialize with no rewrite.
19371        let mut old = serde_json::to_value(record).unwrap();
19372        old.as_object_mut().unwrap().remove("rewrite");
19373        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19374        assert!(old.rewrite.is_none());
19375        let id = record.question.clone().unwrap();
19376        ask::Questions::open()
19377            .update(&id, |q| {
19378                q.choices.push(crate::github_text::USE_REWRITE.into());
19379                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19380            })
19381            .unwrap();
19382        let message = runner
19383            .guarded_pr_message(&winner, None, true)
19384            .await
19385            .unwrap()
19386            .unwrap();
19387        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19388        assert_eq!(
19389            runner.state.github_text.as_ref().unwrap().choice,
19390            Some(GithubTextChoice::Fallback)
19391        );
19392    }
19393
19394    #[tokio::test]
19395    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19396        crate::run::pin_test_home();
19397        let dir = tempfile::tempdir().unwrap();
19398        let mut runner = runner_at(RunStatus::Gating);
19399        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19400        runner.state.repo = dir.path().to_owned();
19401        runner.state.candidates[0].worktree = dir.path().to_owned();
19402        seed_public_summary(&mut runner);
19403        let winner = runner.state.candidates[0].clone();
19404        assert!(
19405            runner
19406                .guarded_pr_message(&winner, None, true)
19407                .await
19408                .unwrap()
19409                .is_none()
19410        );
19411        let store = ask::Questions::open();
19412        let find = |run: &str| {
19413            store
19414                .list()
19415                .into_iter()
19416                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19417                .unwrap()
19418        };
19419        let run = runner.state.id.clone();
19420        let q = find(&run);
19421        let fp = runner
19422            .state
19423            .github_text
19424            .as_ref()
19425            .unwrap()
19426            .fingerprint
19427            .clone();
19428        let name = crate::github_text::artifact_name(&fp);
19429        let path = crate::run::artifact_path(&runner.state, &name);
19430        assert!(q.detail.contains(&path.display().to_string()));
19431        assert!(q.detail.contains("title-language: title"));
19432        let saved = std::fs::read_to_string(&path).expect("artifact written");
19433        assert!(saved.contains("日本語のタイトル"));
19434        assert!(q.cwd.is_none());
19435        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19436        // A bad replacement is refused and asked about once more.
19437        let say = |id: &str, text: &str, choice: &str| {
19438            store
19439                .update(id, |q| {
19440                    q.thread.push(ask::Turn {
19441                        who: ask::Who::Operator,
19442                        body: text.to_owned(),
19443                        at: jiff::Timestamp::now(),
19444                        note: None,
19445                    });
19446                    q.answer(ask::Answer::Choice(choice.to_owned()))
19447                })
19448                .unwrap();
19449        };
19450        say(&q.id, "まだ日本語", "use my text");
19451        assert!(
19452            runner
19453                .guarded_pr_message(&winner, None, true)
19454                .await
19455                .unwrap()
19456                .is_none()
19457        );
19458        let second = find(&run);
19459        assert_ne!(second.id, q.id);
19460        assert!(second.detail.contains("did not pass"));
19461        // The retry still compares the original text; the rejected
19462        // replacement is named by category only.
19463        assert!(second.detail.contains("日本語のタイトル"));
19464        assert!(!second.detail.contains("まだ日本語"));
19465        assert!(second.detail.contains(&path.display().to_string()));
19466        say(&second.id, "fix: retry failed requests", "use my text");
19467        let message = runner
19468            .guarded_pr_message(&winner, None, true)
19469            .await
19470            .unwrap()
19471            .expect("a vetted replacement is posted");
19472        assert_eq!(message.title, "fix: retry failed requests");
19473        assert_eq!(
19474            runner
19475                .state
19476                .github_text
19477                .as_ref()
19478                .unwrap()
19479                .chosen_title
19480                .as_deref(),
19481            Some("fix: retry failed requests")
19482        );
19483        assert!(runner.state.github_text.as_ref().unwrap().resolved);
19484        // Resumed again: the saved decision is applied, nothing is re-asked.
19485        let again = runner
19486            .guarded_pr_message(&winner, None, true)
19487            .await
19488            .unwrap()
19489            .unwrap();
19490        assert_eq!(again.title, message.title);
19491        assert_eq!(
19492            store
19493                .list()
19494                .into_iter()
19495                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19496                .count(),
19497            2
19498        );
19499    }
19500
19501    #[test]
19502    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19503        // What magi itself writes stays English under any configured language,
19504        // so a future localisation of these headings fails here. (The agents'
19505        // own text is also checked by the posting gate.)
19506        let mut state = state_with_summary(
19507            "add retries",
19508            "TITLE: fix(web): batch reads\n- reads run.json once",
19509        );
19510        state.config.graph.language = "ja".to_owned();
19511        let m = pr_message(&state, 'A');
19512        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19513        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19514
19515        // The task is the operator's own text: it goes in untouched, and the
19516        // fallback title (no summary) may be in its language too.
19517        let task = "今回やってほしいこと: results projector を直す";
19518        let mut state = state_with_summary(task, "- no title line");
19519        state.config.graph.language = "ja".to_owned();
19520        let m = pr_message(&state, 'A');
19521        assert_eq!(
19522            m.title,
19523            format!("chore: land candidate A of run {}", state.id)
19524        );
19525        assert!(
19526            m.body.contains(&format!(
19527                "<summary>Original task</summary>\n\n{task}\n\n</details>"
19528            )),
19529            "{}",
19530            m.body
19531        );
19532    }
19533
19534    #[test]
19535    fn pr_message_scrubs_home_paths_and_addresses() {
19536        let state = state_with_summary(
19537            "fix it in /Users/someone/src/x",
19538            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19539        );
19540        let m = pr_message(&state, 'A');
19541        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19542            assert!(!m.body.contains(leak), "{}", m.body);
19543        }
19544        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19545    }
19546
19547    #[test]
19548    fn pr_message_survives_a_task_that_closes_details() {
19549        let state = state_with_summary("a </details> b", "TITLE: fix: x");
19550        let m = pr_message(&state, 'A');
19551        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19552    }
19553
19554    #[test]
19555    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19556        let cmd = manual_merge_command(
19557            MergeStyle::Squash,
19558            Path::new("/repo"),
19559            "b",
19560            "fix: \"quoted\" $(x) `y`\n\nbody",
19561        );
19562        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19563    }
19564
19565    #[test]
19566    fn manual_merge_command_matches_the_configured_style() {
19567        let repo = Path::new("/repo");
19568        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19569
19570        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19571        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19572
19573        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19574        assert_eq!(
19575            squash,
19576            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19577             \"Merge magi run 0832 (candidate A)\""
19578        );
19579
19580        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19581        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19582    }
19583
19584    #[test]
19585    fn a_nudge_gets_a_quarter_of_the_budget() {
19586        // The judge and implement budgets magi ships with.
19587        assert_eq!(retry_budget(secs(1200), true), secs(300));
19588        assert_eq!(retry_budget(secs(3600), true), secs(900));
19589    }
19590
19591    #[test]
19592    fn a_resent_prompt_keeps_the_whole_budget() {
19593        // The seat kept no context, so the retry is the original job again and
19594        // shortening it would only guarantee a second failure.
19595        assert_eq!(retry_budget(secs(1200), false), secs(1200));
19596        assert_eq!(retry_budget(secs(60), false), secs(60));
19597    }
19598
19599    #[test]
19600    fn the_floor_never_exceeds_the_original_budget() {
19601        // A short configured timeout must not be *raised* by the floor: the
19602        // operator asked for a bound, and a retry may not outlast the attempt
19603        // it is retrying.
19604        assert_eq!(retry_budget(secs(60), true), secs(60));
19605        assert_eq!(retry_budget(secs(480), true), secs(120));
19606        assert_eq!(retry_budget(secs(0), true), secs(0));
19607    }
19608
19609    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19610        agent::CommandEvidence {
19611            id: "item1".to_owned(),
19612            description: "cargo test".to_owned(),
19613            exit_code,
19614            result_summary: String::new(),
19615            source: "codex".to_owned(),
19616        }
19617    }
19618
19619    #[test]
19620    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19621        // No evidence is not the same fact as unconfirmed evidence: a
19622        // backend with no adapter, or a reply that ran no commands at all,
19623        // must not be misread as carrying a dangling job.
19624        assert!(!has_unconfirmed_command(&[]));
19625    }
19626
19627    #[test]
19628    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19629        // Deliberately not a check on the exit code's *value*: a fixer
19630        // legitimately runs something that fails mid-iteration before it
19631        // succeeds, and that must never by itself reopen a valid report.
19632        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19633        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19634        assert!(!has_unconfirmed_command(&[
19635            evidence(Some(0)),
19636            evidence(Some(101))
19637        ]));
19638    }
19639
19640    #[test]
19641    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19642        assert!(has_unconfirmed_command(&[
19643            evidence(Some(0)),
19644            evidence(None)
19645        ]));
19646    }
19647
19648    #[test]
19649    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19650        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19651        assert_eq!(
19652            verified_noop_claim(true, &[], text).as_deref(),
19653            Some("already fixed by b32cfc4, on main.")
19654        );
19655    }
19656
19657    #[test]
19658    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19659        // A timeout or a bad exit code reads as the ordinary loss it is,
19660        // whatever the reply's own prose claims.
19661        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19662        assert!(verified_noop_claim(false, &[], text).is_none());
19663    }
19664
19665    #[test]
19666    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19667        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19668        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19669        // A confirmed command alongside the marker is fine.
19670        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19671    }
19672
19673    #[test]
19674    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19675        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19676    }
19677
19678    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
19679    /// pair, in order, labelled A, B, C, ...
19680    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19681        runner.state.candidates = shape
19682            .iter()
19683            .enumerate()
19684            .map(|(i, &(empty, verified))| Candidate {
19685                index: i,
19686                label: (b'A' + i as u8) as char,
19687                agent: "sonnet".to_owned(),
19688                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19689                worktree: PathBuf::from(format!("/wt/{i}")),
19690                summary: String::new(),
19691                stat: String::new(),
19692                files: 0,
19693                commits: 0,
19694                empty,
19695                failed: None,
19696                verified_noop: verified.map(str::to_owned),
19697                duration_ms: 0,
19698                folded: false,
19699            })
19700            .collect();
19701    }
19702
19703    #[test]
19704    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19705        ask_test_home();
19706        let mut runner = runner_at(RunStatus::Implementing);
19707        set_candidates(
19708            &mut runner,
19709            &[
19710                (true, Some("already on main at b32cfc4")),
19711                (true, Some("same fix, see the existing test")),
19712            ],
19713        );
19714
19715        runner
19716            .after_implement()
19717            .expect("a verified no-op is not an error");
19718
19719        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19720    }
19721
19722    #[test]
19723    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19724        ask_test_home();
19725        let mut runner = runner_at(RunStatus::Implementing);
19726        // Candidate A declares a verified no-op; candidate B simply wrote
19727        // nothing and said nothing about why. One candidate's claim is not
19728        // the whole run's agreement.
19729        set_candidates(
19730            &mut runner,
19731            &[(true, Some("already on main at b32cfc4")), (true, None)],
19732        );
19733
19734        let err = runner
19735            .after_implement()
19736            .expect_err("an unverified empty candidate must still fail the run");
19737
19738        assert!(
19739            err.to_string().contains("no candidate produced a change"),
19740            "{err}"
19741        );
19742        assert_eq!(runner.state.status, RunStatus::Failed);
19743    }
19744
19745    #[test]
19746    fn after_implement_still_fails_an_ordinary_all_empty_run() {
19747        ask_test_home();
19748        let mut runner = runner_at(RunStatus::Implementing);
19749        set_candidates(&mut runner, &[(true, None), (true, None)]);
19750
19751        let err = runner
19752            .after_implement()
19753            .expect_err("no candidate declared anything; this is an ordinary failure");
19754
19755        assert!(
19756            err.to_string().contains("no candidate produced a change"),
19757            "{err}"
19758        );
19759        assert_eq!(runner.state.status, RunStatus::Failed);
19760    }
19761}
19762
19763/// A graft rollback whose restore could not be confirmed (see
19764/// [`Runner::graft`]).
19765#[derive(Debug)]
19766struct GraftRestoreFailed(String);
19767
19768impl std::fmt::Display for GraftRestoreFailed {
19769    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19770        write!(
19771            f,
19772            "could not restore the winner to its pre-graft commit: {}",
19773            self.0
19774        )
19775    }
19776}
19777
19778impl std::error::Error for GraftRestoreFailed {}
19779
19780/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
19781/// not the agent: they never draw on an agent's own timeout, and running out of
19782/// them never reads as an agent failure.
19783#[cfg(test)]
19784mod resource_wait_tests {
19785    use super::*;
19786    use std::collections::BTreeMap;
19787
19788    fn sh(id: &str, script: &str) -> AgentSpec {
19789        AgentSpec {
19790            id: id.to_owned(),
19791            kind: crate::config::AgentKind::Command,
19792            model: None,
19793            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19794            extra_args: Vec::new(),
19795            env: BTreeMap::new(),
19796            prompt_delivery: None,
19797            max_turn_secs: None,
19798        }
19799    }
19800
19801    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19802        SeatJob {
19803            seat: SeatState::new("impl-A", &spec.id, 1),
19804            spec,
19805            cwd: dir.to_path_buf(),
19806            prompt: "do it".to_owned(),
19807            timeout,
19808            allow_write: true,
19809            sessions: false,
19810            artifacts: dir.join("art"),
19811            stem: "impl-A".to_owned(),
19812            handover: None,
19813        }
19814    }
19815
19816    fn state(resource_wait_secs: u64) -> RunState {
19817        let mut config = Config::default();
19818        config.graph.timeout_resource_wait = resource_wait_secs;
19819        RunState::new(
19820            PathBuf::from("/repo"),
19821            "main".to_owned(),
19822            "abc1234".to_owned(),
19823            "task".to_owned(),
19824            config,
19825        )
19826    }
19827
19828    fn hold(cache: &Path) -> crate::cache::Guard {
19829        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19830        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19831            crate::cache::AcquireOutcome::Acquired(g) => g,
19832            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19833        }
19834    }
19835
19836    async fn run(
19837        jobs: Vec<SeatJob>,
19838        sem: Arc<Semaphore>,
19839        cache: Option<&Path>,
19840        st: &mut RunState,
19841    ) -> Vec<(usize, SeatState, AgentOutcome)> {
19842        let prompts = Prompts::default();
19843        let ctx = WaveCtx {
19844            run: "r",
19845            node: "implement",
19846            prompts: &prompts,
19847            cache,
19848            round: None,
19849            carry_seats: false,
19850            pauses: None,
19851        };
19852        wave(jobs, sem, &ctx, st, 0).await
19853    }
19854
19855    #[tokio::test]
19856    async fn verification_cache_wait_preserves_the_command_budget() {
19857        crate::run::pin_test_home();
19858        let tmp = tempfile::tempdir().unwrap();
19859        let cache = tmp.path().join("target");
19860        let guard = hold(&cache);
19861        let release = tokio::spawn(async move {
19862            tokio::time::sleep(Duration::from_millis(100)).await;
19863            guard.release();
19864        });
19865        let mut st = state(10);
19866        let budget = Duration::from_millis(50);
19867        let wait = repository_wait(&st, None);
19868        let (outcomes, _) = with_cache_lease(
19869            &mut st,
19870            Some(&cache),
19871            "gate",
19872            "gate",
19873            tmp.path(),
19874            "abc1234",
19875            budget,
19876            "test gate",
19877            &wait,
19878            |_, available| async move {
19879                assert_eq!(
19880                    available, budget,
19881                    "contention must not shorten verification"
19882                );
19883                (Vec::new(), false, Vec::new())
19884            },
19885        )
19886        .await
19887        .unwrap();
19888        release.await.unwrap();
19889        assert!(outcomes.is_empty(), "{outcomes:?}");
19890        assert!(
19891            st.events
19892                .iter()
19893                .any(|event| event.message.contains("waiting for"))
19894        );
19895        assert!(st.resource_wait.is_none());
19896        // The verification body no longer owns the lease after returning.
19897        hold(&cache).release();
19898    }
19899
19900    #[tokio::test]
19901    async fn verification_cache_wait_expires_without_starting_commands() {
19902        crate::run::pin_test_home();
19903        let tmp = tempfile::tempdir().unwrap();
19904        let cache = tmp.path().join("target");
19905        let _held = hold(&cache);
19906        let mut st = state(0);
19907        let wait = repository_wait(&st, None);
19908        let (outcomes, _) = with_cache_lease(
19909            &mut st,
19910            Some(&cache),
19911            "gate",
19912            "gate",
19913            tmp.path(),
19914            "abc1234",
19915            Duration::from_secs(60),
19916            "test gate",
19917            &wait,
19918            |_, _| async { panic!("commands must not start without the cache lease") },
19919        )
19920        .await
19921        .unwrap();
19922        assert_eq!(outcomes.len(), 1);
19923        assert!(outcomes[0].resource_blocked);
19924        assert!(st.resource_wait.is_none());
19925        assert!(matches!(
19926            st.resource_block.unwrap().cause,
19927            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19928        ));
19929    }
19930
19931    #[tokio::test]
19932    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19933        crate::run::pin_test_home();
19934        let tmp = tempfile::tempdir().unwrap();
19935        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19936        let cache = tmp.path().join("cache");
19937        let mut state = state(21600);
19938        let wait = repository_wait(&state, None);
19939        let (outcomes, _) = with_cache_lease(
19940            &mut state,
19941            Some(&cache),
19942            "gate",
19943            "gate",
19944            tmp.path(),
19945            "head",
19946            Duration::from_secs(60),
19947            "gate",
19948            &wait,
19949            |_, _| async { panic!("freshness failure must not start verification") },
19950        )
19951        .await
19952        .unwrap();
19953        assert_eq!(outcomes.len(), 1);
19954        assert!(outcomes[0].resource_blocked);
19955        let block = state.resource_block.unwrap();
19956        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19957        assert!(!block.description().contains("timed out"));
19958    }
19959
19960    #[tokio::test]
19961    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19962        crate::run::pin_test_home();
19963        let tmp = tempfile::tempdir().unwrap();
19964        let cache = tmp.path().join("cache");
19965        let _held = hold(&cache);
19966        let mut state = state(21600);
19967        let pause = Pause::new();
19968        let interrupt = Pause::new();
19969        let cause = ParkCause::Operator;
19970        pause.park_for(cause.clone());
19971        let prompts = Prompts::default();
19972        let ctx = WaveCtx {
19973            run: "test",
19974            node: "fix",
19975            prompts: &prompts,
19976            cache: Some(&cache),
19977            round: None,
19978            carry_seats: false,
19979            pauses: Some((&pause, &interrupt)),
19980        };
19981        let result = wave(
19982            vec![job(
19983                sh("writer", "exit 99"),
19984                tmp.path(),
19985                Duration::from_secs(1),
19986            )],
19987            Arc::new(Semaphore::new(1)),
19988            &ctx,
19989            &mut state,
19990            0,
19991        )
19992        .await;
19993        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19994        assert_eq!(result[0].1.turns, 0);
19995        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19996        assert_eq!(
19997            state.resource_block.unwrap().cause,
19998            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19999        );
20000        assert!(state.active.is_empty());
20001    }
20002
20003    #[tokio::test]
20004    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
20005        crate::run::pin_test_home();
20006        let tmp = tempfile::tempdir().unwrap();
20007        let cache = tmp.path().join("target-a");
20008        let guard = hold(&cache);
20009        // Held for longer than the agent's whole 1s budget.
20010        tokio::spawn(async move {
20011            tokio::time::sleep(Duration::from_millis(1500)).await;
20012            guard.release();
20013        });
20014        let mut st = state(60);
20015        let results = run(
20016            vec![job(
20017                sh("alpha", "echo ok"),
20018                tmp.path(),
20019                Duration::from_secs(1),
20020            )],
20021            Arc::new(Semaphore::new(1)),
20022            Some(&cache),
20023            &mut st,
20024        )
20025        .await;
20026        assert!(
20027            matches!(results[0].2, AgentOutcome::Ok(_)),
20028            "the wait must not be charged to the agent"
20029        );
20030        assert!(st.resource_block.is_none());
20031        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20032        assert!(st.resource_wait.is_none() && st.active.is_empty());
20033    }
20034
20035    #[tokio::test]
20036    #[cfg(unix)]
20037    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20038        use std::os::fd::AsRawFd as _;
20039        crate::run::pin_test_home();
20040        let tmp = tempfile::tempdir().unwrap();
20041        crate::git::git(tmp.path(), &["init"]).await.unwrap();
20042        let common = tmp.path().join(".git");
20043        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20044        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20045        assert_eq!(
20046            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20047            0
20048        );
20049        let mut st = state(1);
20050        st.config.coordination = crate::config::test_coordination();
20051        let mut reader = job(
20052            sh("reader", "echo must-not-run"),
20053            tmp.path(),
20054            Duration::from_secs(30),
20055        );
20056        reader.allow_write = false;
20057        let results = run(
20058            vec![reader.clone()],
20059            Arc::new(Semaphore::new(1)),
20060            None,
20061            &mut st,
20062        )
20063        .await;
20064        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20065        assert_eq!(results[0].1.turns, 0);
20066        assert!(FailClass::of(&results[0].2).is_none());
20067        assert_eq!(
20068            st.resource_block.as_ref().unwrap().kind,
20069            "repository coordination"
20070        );
20071        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20072        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20073        drop(external);
20074        let editor = crate::repo_lock::SeatLease::for_checkout(
20075            tmp.path(),
20076            true,
20077            crate::config::test_coordination().as_ref(),
20078        )
20079        .await
20080        .unwrap();
20081        let pause = Pause::new();
20082        let interrupt = Pause::new();
20083        let prompts = Prompts::default();
20084        let ctx = WaveCtx {
20085            run: "r",
20086            node: "review",
20087            prompts: &prompts,
20088            cache: None,
20089            round: None,
20090            carry_seats: false,
20091            pauses: Some((&pause, &interrupt)),
20092        };
20093        let (results, ()) = tokio::join!(
20094            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20095            async {
20096                tokio::time::sleep(Duration::from_millis(50)).await;
20097                pause.park();
20098            }
20099        );
20100        assert!(
20101            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20102        );
20103        assert_eq!(results[0].1.turns, 0);
20104        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20105        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20106        drop(editor);
20107    }
20108
20109    #[tokio::test]
20110    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20111        crate::run::pin_test_home();
20112        let tmp = tempfile::tempdir().unwrap();
20113        let cache = tmp.path().join("target-b");
20114        let _held = hold(&cache);
20115        let mut st = state(1);
20116        let results = run(
20117            vec![job(
20118                sh("alpha", "echo ok"),
20119                tmp.path(),
20120                Duration::from_secs(1),
20121            )],
20122            Arc::new(Semaphore::new(1)),
20123            Some(&cache),
20124            &mut st,
20125        )
20126        .await;
20127        let (_, seat, out) = &results[0];
20128        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20129        assert!(wave_blocked(&results));
20130        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20131        assert_eq!(seat.turns, 0, "nothing was asked");
20132        assert!(
20133            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20134            "no agent was spawned"
20135        );
20136        assert!(FailClass::of(out).is_none(), "not a failure class");
20137        let block = st.resource_block.as_ref().expect("recorded for the resume");
20138        assert_eq!(block.kind, "shared build cache");
20139        assert!(matches!(
20140            block.cause,
20141            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20142        ));
20143        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20144        assert!(st.active.is_empty());
20145    }
20146
20147    #[tokio::test]
20148    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20149        crate::run::pin_test_home();
20150        let tmp = tempfile::tempdir().unwrap();
20151        let mut st = state(1);
20152        let results = run(
20153            vec![job(
20154                sh("alpha", "echo ok"),
20155                tmp.path(),
20156                Duration::from_secs(1),
20157            )],
20158            Arc::new(Semaphore::new(0)),
20159            None,
20160            &mut st,
20161        )
20162        .await;
20163        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20164        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20165        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20166    }
20167
20168    #[tokio::test]
20169    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20170        crate::run::pin_test_home();
20171        let tmp = tempfile::tempdir().unwrap();
20172        let mut st = state(60);
20173        let results = run(
20174            vec![job(
20175                sh("alpha", "sleep 30"),
20176                tmp.path(),
20177                Duration::from_secs(1),
20178            )],
20179            Arc::new(Semaphore::new(1)),
20180            None,
20181            &mut st,
20182        )
20183        .await;
20184        let out = &results[0].2;
20185        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20186        let class = FailClass::of(out).expect("a failure");
20187        assert_eq!(class, FailClass::Timeout);
20188        assert!(should_hand_over(None, &class), "existing handover policy");
20189        assert!(st.resource_block.is_none());
20190    }
20191}