Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's Git administration access, network and repository coordination \
526         before retrying; an unattended seat cannot approve those permissions"
527    )
528}
529
530/// Outcome of the shared fixer chain, before caller-owned stop policy.
531enum FixStep {
532    Resource,
533    Recorded { report_lost: bool },
534}
535
536/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
537/// on an early return or a panic.
538///
539/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
540/// manual `magi fix` invocations against the same run are otherwise
541/// invisible to each other and would race to remove and recreate the same
542/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
543/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
544/// which is only ever reclaimed later, out of band, by
545/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
546/// `magi fix` invocation is not necessarily running under either of those, so
547/// nothing would ever sweep a lock a killed or crashed process left behind.
548/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
549/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
550/// lease use: an unreadable or unparsable pid, or a liveness query the
551/// platform cannot answer, reads as alive and the lock is left in place.
552struct FixClaim {
553    path: PathBuf,
554}
555
556impl FixClaim {
557    fn acquire(dir: &Path) -> Result<Self> {
558        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559        let path = dir.join("fix.lock");
560        let retry = crate::lock_file::Retry::new(None);
561        match Self::create(&path, retry) {
562            Ok(claim) => Ok(claim),
563            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564                if Self::reclaim_if_dead(&path) {
565                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566                } else {
567                    bail!(
568                        "another repair or `magi fix` is already running for this run ({} exists)",
569                        path.display()
570                    )
571                }
572            }
573            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574        }
575    }
576
577    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578        let mut f = retry.create(path)?;
579        use std::io::Write as _;
580        // Read back by `reclaim_if_dead` on a later, stuck invocation.
581        writeln!(f, "{}", std::process::id())?;
582        Ok(Self {
583            path: path.to_owned(),
584        })
585    }
586
587    /// True if the lock named a process confirmed dead, in which case it was
588    /// also removed. Never true on an unreadable file, an unparsable pid, or
589    /// a liveness query the platform cannot answer — see this type's own doc.
590    fn reclaim_if_dead(path: &Path) -> bool {
591        let dead = std::fs::read_to_string(path)
592            .ok()
593            .and_then(|body| body.trim().parse::<u32>().ok())
594            .is_some_and(|pid| !crate::proc::pid_alive(pid));
595        dead && std::fs::remove_file(path).is_ok()
596    }
597}
598
599impl Drop for FixClaim {
600    fn drop(&mut self) {
601        let _ = std::fs::remove_file(&self.path);
602    }
603}
604
605/// Holds the mutable runner as well as its durable lease: cancellation must
606/// clear both copies of ownership, including state a caller may save later.
607struct DriverExecution<'a> {
608    runner: &'a mut Runner,
609    driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613    fn drop(&mut self) {
614        self.driver.release_local(&mut self.runner.state);
615    }
616}
617
618impl Runner {
619    /// Start a fresh run against `repo`.
620    pub async fn start(
621        repo: &Path,
622        instruction: String,
623        config: Config,
624        origin: Origin,
625    ) -> Result<Self> {
626        Self::start_naming(repo, instruction, "", config, origin).await
627    }
628
629    /// [`Runner::start`] for a queued task. Titles and instructions never
630    /// select implementation sources. The title argument is retained for callers.
631    pub async fn start_naming(
632        repo: &Path,
633        instruction: String,
634        _also_scan: &str,
635        config: Config,
636        origin: Origin,
637    ) -> Result<Self> {
638        let repo = git::toplevel(repo).await?;
639        let missing = agent::missing_programs(&config.agents);
640        if !missing.is_empty() {
641            bail!(
642                "these agent programs are not on PATH: {}. Fix the roster in \
643                 magi.toml or install them.",
644                missing.join(", ")
645            );
646        }
647        let base_branch =
648            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649                .await?;
650        let fetched_base = resolve_base(
651            &repo,
652            &base_branch,
653            &config.merge.remote,
654            config.coordination.as_ref(),
655        )
656        .await?;
657        let base_commit = fetched_base.commit.clone();
658        let roles = config.resolve_roles()?;
659        let max_parallel = config.graph.max_parallel.max(1);
660        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661        // Recorded before the first save, so a crash right after minting
662        // cannot leave a run with no origin. Legibility only: nothing reads it
663        // to decide anything.
664        state.fetched_base = Some(fetched_base);
665        state.origin = Some(origin);
666        state.reserve()?;
667        state.event("start", format!("run {} created", state.id));
668        state.save()?;
669        Ok(Self {
670            state,
671            roles,
672            sem: Arc::new(Semaphore::new(max_parallel)),
673            pause: Pause::new(),
674            interrupt: Pause::new(),
675        })
676    }
677
678    /// Open a review-only run against work that already exists on `branch`.
679    ///
680    /// The expensive half of the graph is the implement wave — measured at
681    /// 111 and 134 internal tool-loop turns on this repository, against a
682    /// handful for a judge or a reviewer. The cheap half is worth running on
683    /// hand-written work too, and there was no way to reach it.
684    ///
685    /// No new state and no schema change are needed: a run with **one** viable
686    /// candidate and a tally already decided degrades `execute` to exactly
687    /// review → gate → merge, because `judge` skips a single-candidate field,
688    /// `deliberate` has fewer than two first choices to reconcile, `vote`
689    /// returns early, `tally` is already present and `fold_losers` has no
690    /// losers. Resuming such a run therefore does the right thing as well.
691    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692        Self::review_taking_over(repo, branch, config, None, origin).await
693    }
694
695    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
696    /// at the same task still has `branch` checked out, its worktree is
697    /// released first if that is safe (see [`crate::handover`]), and the
698    /// review refuses with the reason if it is not. `None` is a hand-run
699    /// review: it has no earlier attempts, so only a worktree of a dead run
700    /// magi recorded itself can be released.
701    pub async fn review_taking_over(
702        repo: &Path,
703        branch: &str,
704        config: Config,
705        takeover: Option<crate::handover::Takeover>,
706        origin: Origin,
707    ) -> Result<Self> {
708        let repo = git::toplevel(repo).await?;
709        let missing = agent::missing_programs(&config.agents);
710        if !missing.is_empty() {
711            bail!(
712                "these agent programs are not on PATH: {}. Fix the roster in \
713                 magi.toml or install them.",
714                missing.join(", ")
715            );
716        }
717        let base_branch =
718            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719                .await?;
720        if base_branch == branch {
721            bail!("`{branch}` is the base branch; there is nothing to review against");
722        }
723        let fetched_base = resolve_base(
724            &repo,
725            &base_branch,
726            &config.merge.remote,
727            config.coordination.as_ref(),
728        )
729        .await?;
730        let base_commit = fetched_base.commit.clone();
731
732        let roles = config.resolve_roles()?;
733        let max_parallel = config.graph.max_parallel.max(1);
734        let mut state = RunState::new(
735            repo.clone(),
736            base_branch,
737            base_commit.clone(),
738            String::new(),
739            config,
740        );
741        state.fetched_base = Some(fetched_base);
742        state.origin = Some(origin);
743        state.reserve()?;
744
745        // Released before anything else touches the branch: a stale local
746        // branch is moved with `git branch -f`, which git refuses while an
747        // earlier attempt's worktree still has it checked out. Everything
748        // after this point that can fail puts the old run back.
749        // A hand-run review has no task, hence no earlier attempts, but a
750        // worktree of a dead run magi made may still be released.
751        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752            earlier: Vec::new(),
753            home: crate::run::home(),
754            choice: None,
755            task_instruction: None,
756        });
757        let wait = repository_wait(&state, None);
758        let coordination_config = state.config.coordination.clone();
759        let coordination =
760            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761                .await?;
762        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763        if let Some(released) = &released {
764            state.event(
765                "release",
766                format!(
767                    "took `{branch}` over from run {}: its worktree was released: {}",
768                    crate::run::short_of(&released.old_id),
769                    released.audit
770                ),
771            );
772        }
773        // The owner's answer to an earlier divergence question is applied
774        // here: after the release (git will not move a checked-out branch)
775        // and before the sync that would otherwise ask again.
776        if let Some(choice) = takeover.choice.as_ref()
777            && let Err(e) =
778                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779                    .await
780        {
781            if let Some(released) = &released {
782                released.restore(&repo, branch).await;
783            }
784            return Err(e.context("applying the owner's answer about the diverged branch"));
785        }
786        // An earlier attempt of this task that published the task head, and
787        // whose work is the branch under review, hands that head over: the
788        // review run updates the same pull request. Without one (a legacy
789        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
790        // own head and its own open pull request is adopted as before.
791        for id in takeover.earlier.iter().rev() {
792            if let Ok(old) = RunState::load_under(id, &takeover.home)
793                && let Some(head) = old.pr_head.clone()
794                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795            {
796                state.pr_head = Some(head);
797                break;
798            }
799        }
800        drop(coordination);
801        let opened = Self::open_review(
802            &repo,
803            branch,
804            state,
805            roles,
806            max_parallel,
807            base_commit,
808            &takeover,
809        )
810        .await;
811        if opened.is_err()
812            && let Some(released) = &released
813        {
814            let _coordination = crate::repo_lock::SeatLease::operation_with(
815                &repo,
816                coordination_config.as_ref(),
817                &wait,
818            )
819            .await?;
820            released.restore(&repo, branch).await;
821        }
822        opened
823    }
824
825    /// Recover exactly the operator-selected public implementation, without touching old sources.
826    pub async fn recover_pr(
827        repo: &Path,
828        task: &str,
829        selection: crate::pr_recovery::Selection,
830        mut config: Config,
831    ) -> Result<Self> {
832        selection.configure(&mut config)?;
833        selection.check_task(task, &selection.instruction)?;
834        let base_branch =
835            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836                .await?;
837        let fetched_base = resolve_base(
838            repo,
839            &base_branch,
840            &config.merge.remote,
841            config.coordination.as_ref(),
842        )
843        .await?;
844        let base_commit = fetched_base.commit.clone();
845        selection
846            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847            .await?;
848        Box::pin(selection.verify_source(repo)).await?;
849        if let Some(proof) = &selection.source_proof {
850            anyhow::ensure!(
851                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852                "source rebase base is outside the fetched landing base"
853            );
854        }
855        let roles = config.resolve_roles()?;
856        let max_parallel = config.graph.max_parallel.max(1);
857        let mut state = RunState::new(
858            repo.to_path_buf(),
859            base_branch,
860            base_commit.clone(),
861            selection.instruction.clone(),
862            config,
863        );
864        state.fetched_base = Some(fetched_base);
865        state.origin = Some(crate::run::Origin::queue(task));
866        state.pr_head = Some(selection.branch.clone());
867        state.pr_recovery = Some(selection.clone());
868        state.reserve()?;
869        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870            // Never leave a reserved record behind as a phantom owner of the head.
871            state.discard_reserved();
872            return Err(e);
873        }
874        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875    }
876
877    async fn open_selected(
878        repo: &Path,
879        state: RunState,
880        roles: ResolvedRoles,
881        max_parallel: usize,
882        base_commit: String,
883        selection: crate::pr_recovery::Selection,
884    ) -> Result<Self> {
885        let mut state = state;
886        let branch = state.branch_for('A');
887        let wait = repository_wait(&state, None);
888        let prepared = async {
889            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890            git::git(repo, &["branch", &branch, selection.source()]).await?;
891            drop(coordination);
892            anyhow::Ok(())
893        }
894        .await;
895        if let Err(e) = prepared {
896            // Nothing was exposed yet: do not leave a reserved record as a
897            // phantom owner of the public head.
898            state.discard_reserved();
899            return Err(e);
900        }
901        state.event(
902            "recovery",
903            format!(
904                "selected implementation {} for {} (public lease {})",
905                selection.source(),
906                selection.url,
907                selection.head
908            ),
909        );
910        let takeover = crate::handover::Takeover {
911            earlier: Vec::new(),
912            home: crate::run::home(),
913            choice: None,
914            task_instruction: Some(selection.instruction.clone()),
915        };
916        if let Err(e) = state.save() {
917            state.discard_reserved();
918            return Err(e);
919        }
920        let id = state.id.clone();
921        let opened = Self::open_review(
922            repo,
923            &branch,
924            state,
925            roles,
926            max_parallel,
927            base_commit,
928            &takeover,
929        )
930        .await;
931        match opened {
932            Ok(runner) => Ok(runner),
933            Err(e) => {
934                if let Ok(mut partial) = RunState::load(&id) {
935                    partial.status = RunStatus::Blocked;
936                    partial.driver_exited = true;
937                    partial.event(
938                        "recovery",
939                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
940                    );
941                    partial.save()?;
942                }
943                Err(crate::pr_recovery::StartupFailed {
944                    run: id,
945                    cause: format!("{e:#}"),
946                }
947                .into())
948            }
949        }
950    }
951
952    /// What the newest earlier attempt that reviewed `branch` leaves for this
953    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
954    /// no earlier attempts. Reads the earlier record only: ownership of its
955    /// worktree and branch is [`crate::handover`]'s business.
956    async fn inherit(
957        repo: &Path,
958        branch: &str,
959        base_commit: &str,
960        takeover: &crate::handover::Takeover,
961    ) -> Option<crate::recovery::RecoveryRecord> {
962        use crate::recovery::{Relation, assess};
963        let newest = takeover.earlier.last()?;
964        let mut chosen = None;
965        for id in takeover.earlier.iter().rev() {
966            if let Ok(old) = RunState::load_under(id, &takeover.home)
967                && !old.reviews.is_empty()
968                && (old.winner().is_some_and(|w| w.branch == branch)
969                    || old.pr_head.as_deref() == Some(branch))
970            {
971                chosen = Some(old);
972                break;
973            }
974        }
975        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976            .await
977            .unwrap_or_default();
978        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979            Some(last) if last.head == tip => Relation::Same,
980            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981            _ => Relation::Diverged,
982        };
983        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984        Some(assess(from, chosen.as_ref(), base_commit, relation))
985    }
986
987    /// The half of [`Runner::review_taking_over`] that can fail after an
988    /// earlier attempt's worktree was released.
989    async fn open_review(
990        repo: &Path,
991        branch: &str,
992        mut state: RunState,
993        roles: ResolvedRoles,
994        max_parallel: usize,
995        base_commit: String,
996        takeover: &crate::handover::Takeover,
997    ) -> Result<Self> {
998        let wait = repository_wait(&state, None);
999        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000        if state.pr_recovery.is_none() {
1001            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002        }
1003        // The commit subjects are the closest thing to a task statement that
1004        // existing work carries, and the reviewers are told as much.
1005        let start = review_base(
1006            repo,
1007            &state.config.merge.remote,
1008            &state.base_branch,
1009            &base_commit,
1010            branch,
1011        )
1012        .await;
1013        let log = git::log_oneline(repo, &start, branch)
1014            .await
1015            .unwrap_or_default();
1016        let log_text = if log.trim().is_empty() {
1017            "(no commit messages)"
1018        } else {
1019            log.trim()
1020        };
1021        let mut instruction = format!(
1022            "Review the work already on branch `{branch}`. There is no task \
1023             statement: what the change claims to do is whatever its commits \
1024             say.\n\n{log_text}"
1025        );
1026        // A retry of a task whose earlier attempt left reviews behind carries
1027        // that attempt's evidence, and the task's own statement with it: the
1028        // completion criteria are the task's, not whatever the commits claim.
1029        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030            if let Some(task) = takeover
1031                .task_instruction
1032                .as_deref()
1033                .filter(|t| !t.trim().is_empty())
1034                && rec.reviewed_head.is_some()
1035            {
1036                instruction = format!(
1037                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038                    task.trim()
1039                );
1040                rec.criteria_inherited = true;
1041            }
1042            state.event("recovery", rec.summary());
1043            state.recovery = Some(rec);
1044        }
1045        if let Some(selected) = &state.pr_recovery {
1046            instruction = selected.instruction.clone();
1047        }
1048        state.task_instruction = takeover.task_instruction.clone();
1049        state.instruction = instruction;
1050        state.reviewed_commits = Some(
1051            git::subjects(repo, &start, branch)
1052                .await
1053                .unwrap_or_default(),
1054        );
1055
1056        // A takeover must not forget a hold that a previous review observed,
1057        // even if repairs have since removed it from the branch.
1058        for id in &takeover.earlier {
1059            if let Ok(old) = RunState::load_under(id, &takeover.home)
1060                && old.repo == state.repo
1061                && (old.pr_head.as_deref() == Some(branch)
1062                    || old.winner().is_some_and(|w| w.branch == branch))
1063            {
1064                for hold in old.branch_owner_holds {
1065                    if !state.branch_owner_holds.contains(&hold) {
1066                        state.event(
1067                            "approval-policy",
1068                            format!(
1069                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070                                hold.control.key(),
1071                                hold.file,
1072                                hold.head
1073                            ),
1074                        );
1075                        state.branch_owner_holds.push(hold);
1076                    }
1077                }
1078            }
1079        }
1080        let reviewed_head = git::rev_parse(repo, branch).await?;
1081        state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083        // An attached worktree, so the fixer's commits land on the branch under
1084        // review rather than on a detached head nobody will look at again.
1085        let worktree = state.worktree_root().join("under-review");
1086        if let Some(parent) = worktree.parent() {
1087            tokio::fs::create_dir_all(parent).await.ok();
1088        }
1089        let path = worktree.to_string_lossy().to_string();
1090        git::git(repo, &["worktree", "add", &path, branch])
1091            .await
1092            .with_context(|| {
1093                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094            })?;
1095
1096        let setup = match ensure_setup_config(&mut state, repo).await {
1097            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098            Err(e) => Err(e),
1099        };
1100        if let Err(e) = setup {
1101            git::worktree_remove_clean(repo, &worktree).await.ok();
1102            return Err(e);
1103        }
1104        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105            .await
1106            .unwrap_or(0);
1107        if commits == 0 {
1108            git::worktree_remove_clean(repo, &worktree).await.ok();
1109            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110        }
1111        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112            .await
1113            .map(|f| f.len())
1114            .unwrap_or(0);
1115        if files == 0
1116            && let (Ok(head_tree), Ok(base_tree)) = (
1117                git::tree_of(&worktree, "HEAD").await,
1118                git::tree_of(&worktree, &base_commit).await,
1119            )
1120            && head_tree == base_tree
1121        {
1122            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123            git::worktree_remove_clean(repo, &worktree).await.ok();
1124            bail!(
1125                "`{branch}` at {} has a tree identical to base {}; this usually means \
1126                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127                 against `{}/{branch}`) rather than an empty change",
1128                short(&head),
1129                short(&base_commit),
1130                state.config.merge.remote
1131            );
1132        }
1133        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134            .await
1135            .unwrap_or_default();
1136
1137        state.candidates.push(Candidate {
1138            index: 0,
1139            label: 'A',
1140            // Not an agent id on purpose: nothing in the roster wrote this, and
1141            // the stats tables must not credit anyone with a win for it.
1142            agent: EXISTING_BRANCH.to_owned(),
1143            branch: branch.to_owned(),
1144            worktree,
1145            summary: String::new(),
1146            stat,
1147            files,
1148            commits,
1149            empty: false,
1150            failed: None,
1151            verified_noop: None,
1152            duration_ms: 0,
1153            folded: false,
1154        });
1155        state.tally = Some(Tally {
1156            first_choice: BTreeMap::from([('A', 0)]),
1157            borda: BTreeMap::new(),
1158            winner: 'A',
1159            rankings: 0,
1160            unanimous_initial: false,
1161            deliberated: false,
1162            changed_votes: 0,
1163            unanimous_final: false,
1164            tie_break: None,
1165            // No panel sat, so no quorum applies. Zero judges is the correct
1166            // number for work that never competed, and must not be reported as
1167            // a collapsed panel.
1168            judges: 0,
1169            present: 0,
1170            quorum: 0,
1171            met_quorum: true,
1172            uncontested: Some("review-only run: nothing competed".to_owned()),
1173        });
1174        state.status = RunStatus::Reviewing;
1175        state.prep_complete = Some(true);
1176        state.event(
1177            "start",
1178            format!(
1179                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180                state.id
1181            ),
1182        );
1183        state.save()?;
1184        Ok(Self {
1185            state,
1186            roles,
1187            sem: Arc::new(Semaphore::new(max_parallel)),
1188            pause: Pause::new(),
1189            interrupt: Pause::new(),
1190        })
1191    }
1192
1193    /// Reopen an existing run.
1194    pub fn resume(id: &str) -> Result<Self> {
1195        let state = RunState::load(id)?;
1196        crate::pr_head::ensure_owner(&state)?;
1197        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198            bail!(
1199                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200            );
1201        }
1202        if let Some(to) = &state.released_to {
1203            bail!(
1204                "run {} cannot be resumed: its worktree was released to run {}",
1205                state.short(),
1206                crate::run::short_of(to)
1207            );
1208        }
1209        let roles = state.config.resolve_roles()?;
1210        if let Some(id) = &state.config.graph.selected_implementer
1211            && state.candidates.iter().any(|c| c.agent != *id)
1212        {
1213            bail!(
1214                "replacement implementer pin does not match the saved candidates; task must stay held"
1215            );
1216        }
1217        let max_parallel = state.config.graph.max_parallel.max(1);
1218        Ok(Self {
1219            state,
1220            roles,
1221            sem: Arc::new(Semaphore::new(max_parallel)),
1222            pause: Pause::new(),
1223            interrupt: Pause::new(),
1224        })
1225    }
1226
1227    /// Walk the graph to a terminal state, skipping nodes already recorded.
1228    ///
1229    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1230    /// the phone - ends here, so this is the one place a run that ended
1231    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1232    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1233    pub async fn execute(&mut self) -> Result<()> {
1234        if let Some(selection) = &self.state.pr_recovery
1235            && (selection.source_proof.is_some() || selection.source_run.is_some())
1236        {
1237            Box::pin(selection.verify_source(&self.state.repo)).await?;
1238        }
1239        // Queue claims cover daemon work; this also serializes manual drivers
1240        // consuming a repair answer, including the new review and decision.
1241        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242            || self.state.land_approval.is_some()
1243            || self.state.merge_decision.is_some()
1244            || !self.state.merge_repairs.is_empty()
1245            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246        {
1247            let claim = FixClaim::acquire(&self.state.dir())?;
1248            let disk = RunState::load(&self.state.id)?;
1249            let checkpoint = |s: &RunState| {
1250                (
1251                    s.merge_repairs.len(),
1252                    s.merge_repairs
1253                        .last()
1254                        .map(|r| (r.question.clone(), r.phase)),
1255                    s.land_approval
1256                        .as_ref()
1257                        .map(|a| (a.question.clone(), a.head.clone())),
1258                )
1259            };
1260            if checkpoint(&disk) != checkpoint(&self.state) {
1261                bail!(
1262                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263                );
1264            }
1265            Some(claim)
1266        } else {
1267            None
1268        };
1269        let mut execution = self.claim_driver().await?;
1270        let runner = &mut *execution.runner;
1271        let wait = runner.repository_wait();
1272        // Keep the graph's large future off the callers' stack, including
1273        // standalone task attempts running on a default-sized test thread.
1274        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275        let result = match result {
1276            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277                record_repository_block(&mut runner.state, &e);
1278                runner.park_here()?;
1279                Ok(())
1280            }
1281            other => other,
1282        };
1283        execution.driver.finish(&mut runner.state).await?;
1284        let ended = if result.is_err() {
1285            Some({
1286                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287                if let Err(error) = &result {
1288                    notice.message = format!(
1289                        "Run {} stopped with an error: {}",
1290                        runner.state.id,
1291                        error.to_string().lines().next().unwrap_or("unknown error")
1292                    );
1293                }
1294                notice
1295            })
1296        } else {
1297            crate::notices::run_ended(&runner.state)
1298        };
1299        if let Some(notice) = ended {
1300            crate::notices::raise(notice);
1301        }
1302        result
1303    }
1304
1305    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307    }
1308
1309    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310        let wait = self.repository_wait();
1311        coordinate_operation(&mut self.state, cwd, &wait).await
1312    }
1313
1314    /// Validate ownership and persist the driver claim in one critical section.
1315    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316        let owner = match crate::pr_head::owner_lock_wait(
1317            &crate::run::home(),
1318            &self.state.id,
1319            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320        )
1321        .await
1322        {
1323            Ok(owner) => owner,
1324            Err(error) => {
1325                let detail = format!("worktree maintenance or ownership update: {error:#}");
1326                // Admission did not own run.json: retain this resource outcome
1327                // in memory for queue settlement without overwriting a concurrent
1328                // retirement or publication transaction.
1329                self.state.resource_block = Some(crate::run::ResourceBlock {
1330                    cause: crate::run::ResourceBlockCause::Other,
1331                    node: "start".into(),
1332                    kind: "worktree maintenance".into(),
1333                    detail: detail.clone(),
1334                    at: Timestamp::now(),
1335                });
1336                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337            }
1338        };
1339        let disk = RunState::load(&self.state.id)?;
1340        crate::pr_head::ensure_owner(&disk)?;
1341        let claimed =
1342            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343        // Only proven-live ownership refuses here. Unknown records (legacy runs
1344        // with no claim marker or process identity) fall through to the
1345        // exclusive execution lease taken just below, which is what actually
1346        // keeps a second driver out.
1347        anyhow::ensure!(
1348            disk.liveness(claimed) != crate::run::Liveness::Live,
1349            "run already has a live driver; wait for its execution to end"
1350        );
1351        if let Some(to) = &disk.released_to {
1352            bail!(
1353                "run {} cannot continue: its worktree was released to run {}",
1354                self.state.short(),
1355                crate::run::short_of(to)
1356            );
1357        }
1358        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359        let mut execution = DriverExecution {
1360            runner: self,
1361            driver,
1362        };
1363        // Declare after the execution guard: on any claim error this short
1364        // lock must drop before that guard reloads and releases ownership.
1365        let _owner = owner;
1366        let state = &mut execution.runner.state;
1367        // Moving again, so it is no longer parked. Set before the walk rather
1368        // than in `resume`, so every way of re-entering the graph clears it
1369        // and a card cannot claim a run is waiting to be resumed while the
1370        // agents are already working.
1371        state.parked = false;
1372        state.resource_wait = None;
1373        state.resource_block = None;
1374        // Any seat this state still lists as answering belongs to whatever
1375        // process last drove this run — this one included, if it crashed
1376        // mid-wave. Cleared and flushed immediately, before anything else
1377        // runs, so a resume can never show a seat as live when nothing is
1378        // asking it anything yet; the node that actually dispatches the next
1379        // wave repopulates it.
1380        state.clear_active();
1381        // Recorded in the same spot, and flushed together with the clear
1382        // above: this is the pid a reader checks (`RunState::liveness`) when
1383        // no daemon claim exists to answer "is a process still driving this
1384        // run" — a plain `magi run` / `magi review` typed into a terminal
1385        // claims nothing there. Always overwritten, never only-if-absent, so
1386        // a resumed run's stale pid from a previous, possibly-dead process
1387        // can never survive into this one's own report. Unlike
1388        // `clear_active`, this changes on every single `execute()` call, so
1389        // the save below is now unconditional rather than only-if-cleared.
1390        //
1391        // `driver_started_at` is recorded in the same breath, from this same
1392        // pid, so `liveness` can tell a live pid that is genuinely still us
1393        // apart from one the OS has since handed to an unrelated process —
1394        // see that field's own doc for why the pid alone is not enough.
1395        let pid = std::process::id();
1396        state.driver_pid = Some(pid);
1397        state.driver_started_at = crate::proc::process_started_at(pid);
1398        state.driver_exited = false;
1399        state.driver_claimed = Some(true);
1400        state.begin_pass();
1401        execution.driver.bind(state);
1402        state.save()?;
1403        execution.driver.record(state)?;
1404        Ok(execution)
1405    }
1406
1407    async fn execute_graph(&mut self) -> Result<()> {
1408        self.state.configure_recovery_resume()?;
1409        // Preservation may invalidate an old empty classification. Resolve it
1410        // before any resume shortcut can reuse a verdict or publish a branch.
1411        self.persist_candidates().await?;
1412        if self
1413            .state
1414            .merge_decision
1415            .as_ref()
1416            .is_some_and(|decision| !decision.closed)
1417            && self.state.status != RunStatus::Landing
1418        {
1419            if crate::merge_decision::resumable_hold(&self.state) {
1420                self.state.merge = Some(MergeOutcome {
1421                    mode: MergeMode::Pr,
1422                    ok: false,
1423                    detail: self
1424                        .state
1425                        .merge_decision
1426                        .as_ref()
1427                        .unwrap()
1428                        .target
1429                        .pr
1430                        .clone(),
1431                    empty: false,
1432                });
1433                self.run_land().await?;
1434                self.settle_questions();
1435                return Ok(());
1436            }
1437            bail!(
1438                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439            );
1440        }
1441        if self.state.config.graph.selected_implementer.is_some()
1442            && self.state.status == RunStatus::Blocked
1443            && self.state.tally.is_none()
1444            && self.state.candidate_preservation.is_empty()
1445            && self.state.interrupted_implementations.is_empty()
1446        {
1447            bail!(
1448                "selected implementation is held; explicitly release the owning task to retry it"
1449            );
1450        }
1451
1452        if self.state.merge_repairs.last().is_some_and(|r| {
1453            matches!(
1454                r.phase,
1455                crate::run::MergeRepairPhase::Requested
1456                    | crate::run::MergeRepairPhase::Fixing
1457                    | crate::run::MergeRepairPhase::Reviewing
1458            )
1459        }) {
1460            if self.repair_merge_review().await? {
1461                self.run_land().await?;
1462            }
1463            self.settle_questions();
1464            return Ok(());
1465        }
1466        // Publication already passed review and gate. Resume only that node:
1467        // syncing to a newer base here could change the verified commit.
1468        if self.state.pr_publication_pending {
1469            self.merge().await?;
1470            if self.resource_halt()? {
1471                return Ok(());
1472            }
1473            self.state.save()?;
1474            self.settle_questions();
1475            return Ok(());
1476        }
1477        // A run that already lost its quorum never resumes into the verdict
1478        // machinery: `deliberate` and `vote` would otherwise clobber the
1479        // stalled marker back to Voting and the run would keep going past a
1480        // verdict that is no longer trustworthy. Everything already recorded is
1481        // kept, so the run stays resumable (or foldable) for a human to pick up.
1482        //
1483        // On --resume the run gets one chance to repair itself: the seats a
1484        // rate limit took out are re-asked. If their quota has since reset and
1485        // the quorum is restored, the run picks up and finishes; otherwise it
1486        // stays stale and still-resumable for a later retry. If it does not
1487        // recover, the returned status stays `Stalled` and nothing was
1488        // clobbered (the recovery only mutates entries for the lost seats).
1489        if self.state.status == RunStatus::Stalled {
1490            if self.recover_stall().await? {
1491                self.finish_after_tally().await?;
1492            } else {
1493                // Still below quorum: persist the marker and stay resumable.
1494                self.state.save()?;
1495            }
1496            return Ok(());
1497        }
1498        // A run parked inside `land` - watching CI, mid fix-round, or
1499        // waiting on the owner's merge approval - resumes directly into it,
1500        // never back through `prep`. Everything before `merge` already
1501        // concluded; that is the only way `status` reaches `Landing` in the
1502        // first place. Re-walking `review_loop` first would also be actively
1503        // wrong: its own status recomputation (see its doc) treats any
1504        // clean round as reason to set `status` to `Gating`, which would
1505        // clobber this marker before `merge` ever ran, and this run would
1506        // never find its way back into `land` at all.
1507        if self.state.status == RunStatus::Landing
1508            || crate::handover::landing_checkpoint(&self.state)
1509        {
1510            self.state.land_sync_pending = true;
1511            self.state.save()?;
1512            self.run_land().await?;
1513            // `run_land` may have settled the run right here - CI came back
1514            // green and the PR merged, say - without ever passing back
1515            // through `merge`'s own trailing call. Whatever it left `status`
1516            // as is what this has to read.
1517            self.settle_questions();
1518            return Ok(());
1519        }
1520        // A run parked at `merge` on the owner's word about a withheld PR text
1521        // resumes straight into `merge`. Re-walking the tail would sync to the
1522        // base again (possibly rebasing, possibly spending the fixer) after the
1523        // gate had passed, and then push a tree nobody reviewed or gated.
1524        if self.state.github_text.is_some()
1525            && self.state.merge.is_none()
1526            && self.state.status == RunStatus::Gating
1527        {
1528            self.merge().await?;
1529            if self.resource_halt()? {
1530                return Ok(());
1531            }
1532            self.state.save()?;
1533            return Ok(());
1534        }
1535        if let Err(e) = self.prep().await {
1536            if e.is::<crate::repo_lock::ResourceBlocked>() {
1537                return Err(e);
1538            }
1539            self.state.status = RunStatus::Blocked;
1540            self.state
1541                .prep_failure
1542                .get_or_insert_with(|| format!("{e:#}"));
1543            self.state
1544                .event("prep", format!("preparation stopped: {e:#}"));
1545            self.state.save()?;
1546            return Err(e);
1547        }
1548        if self.park_here()? {
1549            return Ok(());
1550        }
1551        self.advise().await?;
1552        if self.park_here()? {
1553            return Ok(());
1554        }
1555        self.implement().await?;
1556        if self.park_here()? {
1557            return Ok(());
1558        }
1559        // `after_implement` already saved the state and settled any open
1560        // questions when it set this; nothing later in the graph has
1561        // anything to judge.
1562        if self.state.status == RunStatus::VerifiedNoop {
1563            return Ok(());
1564        }
1565        self.judge().await?;
1566        if self.park_here()? {
1567            return Ok(());
1568        }
1569        self.deliberate().await?;
1570        if self.park_here()? {
1571            return Ok(());
1572        }
1573        self.vote().await?;
1574        if self.park_here()? {
1575            return Ok(());
1576        }
1577        self.tally()?;
1578        // A verdict that lost its quorum is not trustworthy: do not review,
1579        // gate, or merge on it. Everything already done is kept, so the run
1580        // stays resumable (or foldable); the human can replace the agent that
1581        // ran out of quota and pick it up.
1582        if self.state.status == RunStatus::Stalled {
1583            // Persist the stalled marker now — the normal end-of-execute save
1584            // below is below this early return, and without it a resumed run
1585            // would reload a pre-tally status and keep going.
1586            self.state.save()?;
1587            return Ok(());
1588        }
1589        self.finish_after_tally().await?;
1590        Ok(())
1591    }
1592
1593    /// A wave could not start its agents because a shared resource stayed busy
1594    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1595    /// resumable. Unstarted seats have no result; completed peers retain
1596    /// their saved candidate state.
1597    fn stop_for_resource(&mut self) -> Result<()> {
1598        // Resource acquisition observes cancellation inside a node. Complete
1599        // the same durable park as every other node boundary before returning.
1600        if self.pause.parked() || self.interrupt.parked() {
1601            self.park_here()?;
1602            return Ok(());
1603        }
1604        if self.state.resource_block.is_some() {
1605            self.state.status = RunStatus::Blocked;
1606        }
1607        self.state.save()
1608    }
1609
1610    /// Stop walking the graph if a node ended on a resource block.
1611    fn resource_halt(&mut self) -> Result<bool> {
1612        if self.state.resource_block.is_none() {
1613            return Ok(false);
1614        }
1615        self.stop_for_resource()?;
1616        Ok(true)
1617    }
1618
1619    /// Park here if asked to, recording it in the run's own timeline.
1620    ///
1621    /// Returns whether the caller should stop walking the graph. The state is
1622    /// saved either way by the node that just finished; this adds the event so
1623    /// the operator's card says why a run that is neither finished nor moving
1624    /// is sitting where it is.
1625    fn park_here(&mut self) -> Result<bool> {
1626        let parking = self.pause.parked() || self.interrupt.parked();
1627        if !parking && self.state.resource_block.is_some() {
1628            self.stop_for_resource()?;
1629            return Ok(true);
1630        }
1631        // Either handle asking is enough - see `Pause`'s own doc for why
1632        // they are never the same one. `interrupt` is checked second so a
1633        // reason it carries is preferred in the message below over a plain
1634        // shutdown park racing it at the same boundary.
1635        if !self.pause.parked() && !self.interrupt.parked() {
1636            return Ok(false);
1637        }
1638        if self.state.parked {
1639            return Ok(true);
1640        }
1641        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642        if let Some(block) = &mut self.state.resource_block
1643            && matches!(
1644                block.cause,
1645                crate::run::ResourceBlockCause::Cancelled { .. }
1646            )
1647        {
1648            block.cause = crate::run::ResourceBlockCause::Cancelled {
1649                park: cause.clone(),
1650            };
1651        }
1652        let why = match cause.as_ref().map(ParkCause::label) {
1653            Some(reason) => format!(
1654                "parked after `{}` ({reason}) — resume to carry on from here",
1655                self.state.status.as_str()
1656            ),
1657            None => format!(
1658                "parked after `{}` — resume to carry on from here",
1659                self.state.status.as_str()
1660            ),
1661        };
1662        self.state.event("park", why);
1663        self.state
1664            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665        self.state.parked = true;
1666        self.state.save()?;
1667        Ok(true)
1668    }
1669
1670    /// Only a new explicit operator approval may retry failed preparation.
1671    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672        if self.state.prep_failure.is_none() {
1673            return Ok(());
1674        }
1675        let failed_at = self
1676            .state
1677            .events
1678            .iter()
1679            .rev()
1680            .find(|event| event.node == "prep")
1681            .map_or(self.state.updated_at, |event| event.at);
1682        anyhow::ensure!(
1683            approval_at > failed_at,
1684            "preparation retry requires a new explicit release after the failure"
1685        );
1686        self.state.prep_failure = None;
1687        self.state.prep_complete = Some(false);
1688        self.state
1689            .event("prep-retry", "operator approved retry of preparation");
1690        Ok(())
1691    }
1692
1693    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1694    pub fn on_pause(&mut self, pause: Pause) {
1695        self.pause = pause;
1696    }
1697
1698    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1699    /// scheduler asking this one run - and no other - to park so a task
1700    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1701    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1702    /// handle.
1703    pub fn watch_interrupt(&mut self, pause: Pause) {
1704        self.interrupt = pause;
1705    }
1706
1707    /// Abandon this run's own open questions, once `status` has actually
1708    /// settled rather than merely paused.
1709    ///
1710    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1711    /// either back up with the candidates, the review round and the seat
1712    /// sessions already on disk, so a question an implementer asked mid-round
1713    /// may still get a real answer read by a real resume. Only the statuses
1714    /// `resumable` excludes are actually final: the run merged, it reached
1715    /// `Ready` with nothing left to do, it failed outright with no
1716    /// established point to continue from, or every candidate agreed, with
1717    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1718    /// every one of those the seat that asked is gone for good, exactly like
1719    /// the run being deleted under `magi run rm` - so the same cleanup
1720    /// applies, worded for what actually happened instead of "the run was
1721    /// deleted".
1722    ///
1723    /// Best-effort and silent on success: called from every place `status`
1724    /// can land on one of those three, including ones a resumed run revisits,
1725    /// so it must cost nothing when there was nothing open to begin with.
1726    fn settle_questions(&mut self) {
1727        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729        }
1730    }
1731
1732    /// The tail of the graph after a trustworthy tally: fold losers, review,
1733    /// gate, merge, and persist.
1734    async fn finish_after_tally(&mut self) -> Result<()> {
1735        // Before the losers are folded: their branches are what the graft
1736        // reads from.
1737        self.graft().await?;
1738        if self.resource_halt()? {
1739            return Ok(());
1740        }
1741        self.fold_losers().await?;
1742        // Before review starts, and again right before the gate: a run's
1743        // review rounds can themselves take long enough for the base to move
1744        // a second time, and the gate is the one node whose "green" gets
1745        // acted on.
1746        self.sync_to_base().await?;
1747        if self.resource_halt()? {
1748            return Ok(());
1749        }
1750        if self.state.status == RunStatus::AlreadyInBase {
1751            return Ok(());
1752        }
1753        self.review_loop().await?;
1754        if self.resource_halt()? {
1755            return Ok(());
1756        }
1757        self.sync_to_base().await?;
1758        if self.resource_halt()? {
1759            return Ok(());
1760        }
1761        if self.state.status == RunStatus::AlreadyInBase {
1762            return Ok(());
1763        }
1764        self.gate().await?;
1765        self.merge().await?;
1766        if self.resource_halt()? {
1767            return Ok(());
1768        }
1769        self.state.save()?;
1770        Ok(())
1771    }
1772
1773    // ---------------------------------------------------------------- prep
1774
1775    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1776    /// run with the step and its output in the event; it is never swallowed.
1777    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1778    /// resume starts over instead of finding candidates and missing seats.
1779    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780        let result = match ensure_setup_config(&mut self.state, repo).await {
1781            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782            Err(e) => Err(e),
1783        };
1784        if let Err(e) = result {
1785            git::worktree_remove_clean(repo, wt).await.ok();
1786            if in_prep {
1787                self.rollback_prep(repo).await;
1788            }
1789            self.state.status = RunStatus::Blocked;
1790            self.state.event(
1791                "setup",
1792                format!("worktree setup failed in {}: {e:#}", wt.display()),
1793            );
1794            self.state.save()?;
1795            return Err(e);
1796        }
1797        Ok(())
1798    }
1799
1800    /// Undo everything `prep` created, so a resume re-runs it whole.
1801    async fn rollback_prep(&mut self, repo: &Path) {
1802        let root = self.state.worktree_root();
1803        for c in std::mem::take(&mut self.state.candidates) {
1804            if matches!(
1805                git::worktree_remove_clean(repo, &c.worktree).await,
1806                Ok(true)
1807            ) {
1808                git::branch_delete(repo, &c.branch).await.ok();
1809            } else {
1810                self.state.candidates.push(c);
1811            }
1812        }
1813        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814        for j in 1..=seats {
1815            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816                .await
1817                .ok();
1818        }
1819        for k in 1..=self.state.config.graph.advisors {
1820            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821                .await
1822                .ok();
1823        }
1824    }
1825
1826    async fn prep(&mut self) -> Result<()> {
1827        if let Some(reason) = &self.state.prep_failure {
1828            bail!(
1829                "candidate preparation previously failed: {reason}; correct the source before retrying"
1830            );
1831        }
1832        if self
1833            .state
1834            .prep_complete
1835            .unwrap_or(!self.state.candidates.is_empty())
1836        {
1837            return Ok(());
1838        }
1839        let repo = self.state.repo.clone();
1840        // Includes rollback, config, refs, checkout creation and setup. No
1841        // repository mutation may precede the real shared lock acquisition.
1842        let _coordination = self.coordinate(&repo).await?;
1843        let base = self.state.base_commit.clone();
1844        let plan = self.state.sources.plan(&repo, &base).await?;
1845        self.rollback_prep(&repo).await;
1846        if !self.state.candidates.is_empty() {
1847            bail!(
1848                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849            );
1850        }
1851        self.state.prep_complete = Some(false);
1852        self.state.status = RunStatus::Prep;
1853        self.state.save()?;
1854        let repo = self.state.repo.clone();
1855        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856        let root = self.state.worktree_root();
1857        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859        // The hook is the write-time half of the blindness contract; the
1860        // presentation filter in `blind` is the half that cannot be bypassed.
1861        let hooks_dir = self.state.dir().join("hooks");
1862        if self.state.config.blind.commit_msg_hook {
1863            std::fs::create_dir_all(&hooks_dir)
1864                .with_context(|| format!("create {}", hooks_dir.display()))?;
1865            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866            let path = hooks_dir.join("commit-msg");
1867            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868            make_executable(&path)?;
1869            // Ref-counted rather than a plain idempotent set: with more than
1870            // one run able to be in flight in the same repository at once
1871            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1872            // true?" check cannot tell "another run of mine still needs
1873            // this" from "nobody does", and the run that happens to finish
1874            // first would disable the hook out from under a sibling still
1875            // relying on it.
1876            git::acquire_worktree_config(&repo).await?;
1877            self.state.enabled_worktree_config = true;
1878        }
1879
1880        // `[worktree] setup` hides its products per worktree, which needs the
1881        // same per-worktree config the hook does. Held until fold, like it.
1882        ensure_setup_config(&mut self.state, &repo).await?;
1883
1884        for (index, (spec, label)) in self
1885            .roles
1886            .implementers
1887            .clone()
1888            .into_iter()
1889            .zip(labels)
1890            .enumerate()
1891        {
1892            let branch = self.state.branch_for(label);
1893            let worktree = root.join(format!("cand-{label}"));
1894            // Persist intent before Git can leave a partially created tree.
1895            // Candidate registration follows setup, so it cannot inventory
1896            // crashes or refusals during creation itself.
1897            self.state.event(
1898                "prep-intent",
1899                serde_json::json!({
1900                    "branch": branch,
1901                    "worktree": worktree,
1902                    "start": start,
1903                    "picks": plan.picks,
1904                })
1905                .to_string(),
1906            );
1907            self.state.save()?;
1908            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909            if self.state.config.blind.commit_msg_hook {
1910                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911            }
1912            git::local_exclude(&worktree, "/.magi/").await?;
1913            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914                // Nothing of this prep may survive: a resume must start over,
1915                // not find half the candidates and skip the rest.
1916                git::worktree_remove_clean(&repo, &worktree).await.ok();
1917                git::branch_delete(&repo, &branch).await.ok();
1918                self.rollback_prep(&repo).await;
1919                self.state.status = RunStatus::Blocked;
1920                self.state
1921                    .event("prep", format!("worktree setup failed: {e:#}"));
1922                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923                self.state.save()?;
1924                return Err(e);
1925            }
1926            self.state.candidates.push(Candidate {
1927                index,
1928                label,
1929                agent: spec.id.clone(),
1930                branch,
1931                worktree,
1932                summary: String::new(),
1933                stat: String::new(),
1934                files: 0,
1935                commits: 0,
1936                empty: false,
1937                failed: None,
1938                verified_noop: None,
1939                duration_ms: 0,
1940                folded: false,
1941            });
1942            self.state.save()?;
1943            for pick in &plan.picks {
1944                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945                {
1946                    self.state.status = RunStatus::Blocked;
1947                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948                    self.state
1949                        .event("prep", format!("cannot apply referenced commit: {e}"));
1950                    self.state.save()?;
1951                    return Err(e);
1952                }
1953            }
1954        }
1955
1956        for j in 1..=self.roles.judges.len() {
1957            let wt = root.join(format!("judge-{j}"));
1958            if !wt.exists() {
1959                git::worktree_add_detached(&repo, &wt, &base).await?;
1960                self.setup_seat_worktree(&repo, &wt, true).await?;
1961            }
1962        }
1963
1964        // Disposable, detached checkouts for the design-deliberation stage's
1965        // advisor seats — the same shape as the judges' above, at the same
1966        // base commit, since advisors also only ever read. Sized off the
1967        // configured count directly rather than a resolved roster: unlike
1968        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1969        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1970        // `prep` has no `ResolvedRoles` field to read a count from here.
1971        if self.state.config.graph.advise {
1972            for k in 1..=self.state.config.graph.advisors {
1973                let wt = root.join(format!("advisor-{k}"));
1974                if !wt.exists() {
1975                    git::worktree_add_detached(&repo, &wt, &base).await?;
1976                    self.setup_seat_worktree(&repo, &wt, true).await?;
1977                }
1978            }
1979        }
1980
1981        // A judge cannot tell it is looking at its own patch — the seats keep
1982        // separate conversations — but a panel that shares agents with the
1983        // field is less independent than it looks, and that is worth saying out
1984        // loud once per run rather than leaving it in the config.
1985        let authors: Vec<&str> = self
1986            .roles
1987            .implementers
1988            .iter()
1989            .map(|a| a.id.as_str())
1990            .collect();
1991        let overlap: Vec<String> = self
1992            .roles
1993            .judges
1994            .iter()
1995            .enumerate()
1996            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998            .collect();
1999        if !overlap.is_empty() {
2000            let note = format!(
2001                "{} also authored a candidate; blind, but the panel is less \
2002                 independent than {} distinct agents would be",
2003                overlap.join(", "),
2004                self.roles.judges.len()
2005            );
2006            self.state.event("prep", note);
2007        }
2008
2009        self.state.event(
2010            "prep",
2011            format!(
2012                "{} candidates, {} judges, base {} ({})",
2013                self.state.candidates.len(),
2014                self.roles.judges.len(),
2015                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016                self.state.base_branch
2017            ),
2018        );
2019        self.state.prep_complete = Some(true);
2020        self.state.status = RunStatus::Implementing;
2021        self.state.save()?;
2022        Ok(())
2023    }
2024
2025    // -------------------------------------------------------------- advise
2026
2027    /// The design-deliberation stage: independent, read-only advisor seats
2028    /// each sketch a design before any implementer touches the repository,
2029    /// and (when at least one produced a usable proposal) a synthesis seat
2030    /// blends them into a brief `implement` carries in every candidate's
2031    /// prompt.
2032    ///
2033    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2034    /// advisors` is the proposal count. Everything here is best-effort and
2035    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2036    /// that cannot reach quota, or a synthesis seat that produced nothing
2037    /// usable all leave `implement` exactly as it was before this stage
2038    /// existed — the task instruction alone — rather than failing the whole
2039    /// competition over an enrichment stage. Every outcome is still recorded
2040    /// as an event, so a run that got nothing from this stage says why.
2041    ///
2042    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2043    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2044    /// resumed run whose stage failed would re-run it, and re-spend the
2045    /// agent calls, on every reentry before `implement`.
2046    ///
2047    /// Also skipped once any candidate shows implementation progress — the
2048    /// exact predicate `implement` itself uses to decide a candidate is no
2049    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2050    /// is not enough: a run created by an older binary that predates this
2051    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2052    /// an already-`Implementing`-or-later run under this build would
2053    /// otherwise walk straight back through `prep` (a no-op once candidates
2054    /// exist) into this node and spawn every advisor seat against worktrees
2055    /// `prep` never recreated — after implementation has already started,
2056    /// which is exactly the invariant this stage exists to guarantee.
2057    async fn advise(&mut self) -> Result<()> {
2058        let implement_untouched = self
2059            .state
2060            .candidates
2061            .iter()
2062            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063        if !self.state.config.graph.advise || self.state.advise_attempted {
2064            return Ok(());
2065        }
2066        if !implement_untouched {
2067            self.state.event(
2068                "advise",
2069                "skipping the design-deliberation stage: at least one \
2070                 candidate already shows implementation progress, so this \
2071                 run is past the point the stage exists to run before"
2072                    .to_owned(),
2073            );
2074            self.state.advise_attempted = true;
2075            self.state.save()?;
2076            return Ok(());
2077        }
2078        let run_id = self.state.id.clone();
2079        let prompts = self.state.config.prompts.clone();
2080        let instruction = self.state.instruction.clone();
2081        let language = self.state.config.graph.language.clone();
2082        let root = self.state.worktree_root();
2083        let n = self.state.config.graph.advisors;
2084        let where_recorded = self.state.dir().join("run.json");
2085
2086        let seats = match self.state.config.advisors() {
2087            Ok(seats) if !seats.is_empty() => seats,
2088            Ok(_) => {
2089                self.state.event(
2090                    "advise",
2091                    format!(
2092                        "[graph] advisors is 0; skipping the design-deliberation \
2093                         stage and continuing without a synthesis brief (see {})",
2094                        where_recorded.display()
2095                    ),
2096                );
2097                self.state.advise_attempted = true;
2098                self.state.save()?;
2099                return Ok(());
2100            }
2101            Err(e) => {
2102                self.state.event(
2103                    "advise",
2104                    format!(
2105                        "could not resolve advisor seats ({e:#}); continuing \
2106                         without a design-deliberation brief (see {})",
2107                        where_recorded.display()
2108                    ),
2109                );
2110                self.state.advise_attempted = true;
2111                self.state.save()?;
2112                return Ok(());
2113            }
2114        };
2115
2116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117        let artifacts = agent::artifacts_dir(&self.state.dir());
2118        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120        let mut jobs = Vec::new();
2121        for (i, spec) in seats.iter().cloned().enumerate() {
2122            let seat_key = format!("advisor-{}", i + 1);
2123            let seat = self.seat(&seat_key, &spec.id);
2124            jobs.push(SeatJob {
2125                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126                spec,
2127                seat,
2128                cwd: worktrees[i % worktrees.len()].clone(),
2129                timeout,
2130                allow_write: false,
2131                sessions: false,
2132                artifacts: artifacts.clone(),
2133                stem: seat_key,
2134                handover: None,
2135            });
2136        }
2137
2138        self.state.event(
2139            "advise",
2140            format!(
2141                "{} advisor seat(s) sketching a design in parallel",
2142                jobs.len()
2143            ),
2144        );
2145        let mut quota_losses = Vec::new();
2146        let cache = self.state.config.cache_dir();
2147        let ctx = WaveCtx {
2148            carry_seats: false,
2149            pauses: Some((&self.pause, &self.interrupt)),
2150            run: &run_id,
2151            node: "advise",
2152            prompts: &prompts,
2153            cache: cache.as_deref(),
2154            round: None,
2155        };
2156        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157        let results = ask_json_wave::<Proposal>(
2158            jobs,
2159            Arc::clone(&self.sem),
2160            self.state.config.graph.retries,
2161            &advisor_roster,
2162            &ctx,
2163            &mut quota_losses,
2164            &mut self.state,
2165            &|p: &Proposal| p.validate(),
2166        )
2167        .await;
2168        if self.state.resource_block.is_some() {
2169            return self.stop_for_resource();
2170        }
2171        self.state.quota.extend(quota_losses);
2172
2173        let mut records = Vec::with_capacity(results.len());
2174        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175            let agent_id = seat.agent.clone();
2176            self.state.seats.insert(seat.key.clone(), seat);
2177            match res {
2178                Ok((proposal, out)) => {
2179                    self.state
2180                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2181                    records.push(advise::AdvisorRecord::proposed(
2182                        i + 1,
2183                        agent_id,
2184                        proposal,
2185                        out.duration_ms,
2186                    ));
2187                }
2188                Err(e) => {
2189                    self.state.event(
2190                        "advise",
2191                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192                    );
2193                    records.push(advise::AdvisorRecord::failed(
2194                        i + 1,
2195                        agent_id,
2196                        e.to_string(),
2197                    ));
2198                }
2199            }
2200        }
2201
2202        let mut advice = advise::Advice {
2203            records,
2204            synthesis: None,
2205        };
2206        if advice.proposals().is_empty() {
2207            self.state.event(
2208                "advise",
2209                "no advisor produced a usable proposal; continuing without a \
2210                 synthesis brief"
2211                    .to_owned(),
2212            );
2213        } else {
2214            match self
2215                .synthesize_brief(
2216                    &advice,
2217                    &instruction,
2218                    &language,
2219                    &worktrees[0],
2220                    &artifacts,
2221                    &run_id,
2222                    &prompts,
2223                    cache.as_deref(),
2224                )
2225                .await
2226            {
2227                Ok(Some(text)) => {
2228                    self.state.event(
2229                        "advise",
2230                        "synthesized a design brief for the implementer".to_owned(),
2231                    );
2232                    advice.synthesis = Some(text);
2233                }
2234                Ok(None) => {
2235                    self.state.event(
2236                        "advise",
2237                        "the synthesis seat produced nothing usable; continuing \
2238                         without a design brief"
2239                            .to_owned(),
2240                    );
2241                }
2242                Err(e) => {
2243                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2244                        return Err(e);
2245                    }
2246                    self.state.event(
2247                        "advise",
2248                        format!("could not synthesize a design brief: {e:#}"),
2249                    );
2250                }
2251            }
2252        }
2253        advise::apply_reflection(&mut advice);
2254
2255        self.state.advice = Some(advice);
2256        self.state.advise_attempted = true;
2257        self.state.save()?;
2258        Ok(())
2259    }
2260
2261    /// The synthesis seat: reads every advisor's proposal and blends them
2262    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2263    /// out of [`Runner::advise`] only for readability — it is not called
2264    /// anywhere else.
2265    ///
2266    /// Picked the same way [`crate::talk`]'s standing conversation and
2267    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2268    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2269    /// order (a claude seat, else the first runnable agent in roster order)
2270    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2271    /// in [`crate::config`] for why a dedicated field exists here at all.
2272    #[allow(clippy::too_many_arguments)]
2273    async fn synthesize_brief(
2274        &mut self,
2275        advice: &advise::Advice,
2276        instruction: &str,
2277        language: &str,
2278        cwd: &Path,
2279        artifacts: &Path,
2280        run_id: &str,
2281        prompts: &Prompts,
2282        cache: Option<&Path>,
2283    ) -> Result<Option<String>> {
2284        let chain = agent::pick_chain(
2285            &self.state.config.agents,
2286            self.state.config.roles.synthesizer.as_ref(),
2287            &agent::installed,
2288            "synthesizer",
2289        )?;
2290        let proposals = advice.proposals();
2291        let mut prompt = prompt::with_overlay(
2292            prompt::synthesize_brief(instruction, &proposals, language),
2293            prompts.overlay("advise"),
2294        );
2295        if cache.is_some() {
2296            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2297            // below — see `prompt::build_cache_note`'s doc for why telling a
2298            // read-only seat to build through the shared cache is exactly how
2299            // a sandbox's write refusal gets misread as a defect.
2300            prompt.push('\n');
2301            prompt.push_str(&prompt::build_cache_note("advise", false));
2302        }
2303        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304        // Each id is tried once, in order; a quota hit, error or unusable
2305        // answer moves to the next. The seat is single-turn (`sessions:
2306        // false`) and the prompt is the whole context, so a fallback agent
2307        // needs nothing carried over.
2308        let mut last = None;
2309        for (n, spec) in chain.iter().enumerate() {
2310            if n > 0 {
2311                self.state
2312                    .event("advise", format!("synthesis falling back to {}", spec.id));
2313            }
2314            let mut seat = self.seat("advise-synthesis", &spec.id);
2315            let wait = self.repository_wait();
2316            let outcome = wait
2317                .scope(agent::invoke(
2318                    spec,
2319                    &mut seat,
2320                    &Invocation {
2321                        scope: crate::agent::InvocationScope::Repository,
2322                        coordination: self.state.config.coordination.as_ref(),
2323                        fetched_base: None,
2324                        cwd,
2325                        prompt: &prompt,
2326                        timeout,
2327                        allow_write: false,
2328                        unsandboxed: false,
2329                        sessions: false,
2330                        artifacts,
2331                        stem: &if n == 0 {
2332                            "advise-synthesis".to_owned()
2333                        } else {
2334                            format!("advise-synthesis-{}", spec.id)
2335                        },
2336                        run: run_id,
2337                        node: "advise",
2338                        cache_dir: None,
2339                        attachments: &[],
2340                        writable: &[],
2341                    },
2342                ))
2343                .await;
2344            if outcome.is_ok() {
2345                self.state.seats.insert(seat.key.clone(), seat);
2346            }
2347            let advance = agent::chain_advances(&outcome);
2348            last = Some(outcome);
2349            if !advance {
2350                break;
2351            }
2352        }
2353        // Exhausted: the last attempt's result is what a single failed seat
2354        // would have produced.
2355        let out = last.expect("a chain holds at least one agent")?;
2356        if !out.usable() {
2357            return Ok(None);
2358        }
2359        let text =
2360            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2361        Ok((!text.trim().is_empty()).then_some(text))
2362    }
2363
2364    // ----------------------------------------------------------- implement
2365
2366    async fn implement(&mut self) -> Result<()> {
2367        self.persist_candidates().await?;
2368        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2369        // agent files with `magi task add` name the run that paid for it. The
2370        // prompt overlay is cloned alongside it because the waves borrow it
2371        // while `self` is mutably borrowed by the node's own bookkeeping.
2372        let run_id = self.state.id.clone();
2373        let prompts = self.state.config.prompts.clone();
2374        let todo: Vec<usize> = self
2375            .state
2376            .candidates
2377            .iter()
2378            .enumerate()
2379            .filter(|(_, c)| {
2380                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2381                    && c.failed.is_none()
2382                    && !c.empty
2383            })
2384            .map(|(i, _)| i)
2385            .collect();
2386        if todo.is_empty() {
2387            return self.after_implement();
2388        }
2389        self.state.status = RunStatus::Implementing;
2390
2391        let language = self.state.config.graph.language.clone();
2392        let github_language = self.state.config.graph.github_language.clone();
2393        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2394        let sessions = self.state.config.graph.sessions;
2395        let artifacts = agent::artifacts_dir(&self.state.dir());
2396        // The design-deliberation stage's blended brief, when `advise` found
2397        // one — carried into every implementer's prompt the same way
2398        // regardless of which candidate it is.
2399        let brief = self
2400            .state
2401            .advice
2402            .as_ref()
2403            .and_then(|a| a.synthesis.as_deref())
2404            .map(str::to_owned);
2405        let attachments = self.state.attachments.clone();
2406
2407        let mut jobs = Vec::new();
2408        for &i in &todo {
2409            let (index, label, worktree) = {
2410                let c = &self.state.candidates[i];
2411                (c.index, c.label, c.worktree.clone())
2412            };
2413            let spec = self.roles.implementers[index].clone();
2414            let seat_key = format!("impl-{label}");
2415            let seat = self.seat(&seat_key, &spec.id);
2416            let instruction = seeded_instruction(&self.state);
2417            jobs.push(SeatJob {
2418                spec,
2419                seat,
2420                prompt: prompt::implement_in(
2421                    &instruction,
2422                    &worktree.to_string_lossy(),
2423                    &language,
2424                    &github_language,
2425                    brief.as_deref(),
2426                    &attachments,
2427                ),
2428                cwd: worktree,
2429                timeout,
2430                allow_write: true,
2431                sessions,
2432                artifacts: artifacts.clone(),
2433                stem: format!("impl-{label}"),
2434                handover: None,
2435            });
2436        }
2437
2438        self.state.event(
2439            "implement",
2440            format!("{} candidates in parallel", jobs.len()),
2441        );
2442        // Kept so a seat whose CLI hung up can be asked again from the same
2443        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2444        // can update a seat's own entry once a fallback agent takes it over —
2445        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2446        // whichever agent actually answered, not the one that quota'd out.
2447        let mut sent = jobs.clone();
2448        let cache = self.state.config.cache_dir();
2449        let ctx = WaveCtx {
2450            carry_seats: false,
2451            pauses: Some((&self.pause, &self.interrupt)),
2452            run: &run_id,
2453            node: "implement",
2454            prompts: &prompts,
2455            cache: cache.as_deref(),
2456            round: None,
2457        };
2458        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2459        if !wave_blocked(&results) {
2460            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2461                .await;
2462            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2463                .await?;
2464            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2465                .await;
2466        }
2467        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2468
2469        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2470            && results
2471                .iter()
2472                .any(|(_, _, out)| FailClass::of(out).is_some());
2473        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2474            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2475                // Executed turns interrupted by disk pressure already carry
2476                // a preservation barrier from wave. Admission-only waits do
2477                // not. Keep the interrupted seat's session for recovery too.
2478                if self
2479                    .state
2480                    .candidate_preservation
2481                    .contains_key(&self.state.candidates[i].label)
2482                {
2483                    self.state.seats.insert(seat.key.clone(), seat);
2484                }
2485                continue;
2486            }
2487            let seat_key = seat.key.clone();
2488            // A quota fallback (`resume_seat_handovers`) may have handed this
2489            // seat to a different agent than the one `prep` recorded on the
2490            // candidate; the stats tables and any later fixer-defaults-to-
2491            // winner's-author lookup must credit whoever actually answered —
2492            // unless every fallback also quota'd out, in which case nobody
2493            // actually answered and crediting the last agent tried would
2494            // erase every earlier agent's own quota loss from the stats
2495            // tables instead of just this one seat's.
2496            let agent = seat.agent.clone();
2497            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2498            self.state.seats.insert(seat.key.clone(), seat);
2499            let label = self.state.candidates[i].label;
2500            self.state.interrupted_implementations.remove(&label);
2501
2502            let (summary, duration, failed, verified_claim) = match out {
2503                AgentOutcome::Ok(o) => {
2504                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2505                    let failed = (!o.usable()).then(|| {
2506                        if o.timed_out {
2507                            "agent timed out".to_owned()
2508                        } else {
2509                            format!(
2510                                "agent exited with {}",
2511                                crate::notices::exit_status(o.exit_code)
2512                            )
2513                        }
2514                    });
2515                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2516                    (text, o.duration_ms, failed, verified_claim)
2517                }
2518                // Left un-resumed by `resume_undelivered` (a dirty tree
2519                // already rescues the work, or there was no session left to
2520                // resume into) — reported like the ordinary failure it is,
2521                // never as if `o.text` (the CLI's raw error JSON) were an
2522                // answer.
2523                AgentOutcome::Dropped(o) => {
2524                    let why = o
2525                        .dropped
2526                        .as_ref()
2527                        .map(|d| d.why.as_str())
2528                        .unwrap_or("the CLI ended the stream without delivering its answer");
2529                    (
2530                        String::new(),
2531                        o.duration_ms,
2532                        Some(format!("the CLI dropped the stream ({why})")),
2533                        None,
2534                    )
2535                }
2536                AgentOutcome::Quota(o) => {
2537                    self.state.quota.push(QuotaLoss {
2538                        seat: seat_key,
2539                        node: "implement".to_owned(),
2540                        at: Timestamp::now(),
2541                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2542                    });
2543                    (
2544                        String::new(),
2545                        o.duration_ms,
2546                        Some("rate limited (quota); produced no change".to_owned()),
2547                        None,
2548                    )
2549                }
2550                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2551                    (String::new(), 0, Some(e), None)
2552                }
2553            };
2554
2555            // Save every completed turn before any fallible Git operation. A failed
2556            // rescue must not erase the reply or prevent collection of other seats.
2557            let c = &mut self.state.candidates[i];
2558            if !exhausted_the_fallback_chain {
2559                c.agent = agent;
2560            }
2561            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2562            c.duration_ms = duration;
2563            c.failed = failed;
2564            c.verified_noop = verified_claim;
2565            self.state
2566                .candidate_preservation
2567                .insert(label, "awaiting persistence".to_owned());
2568        }
2569        self.state.save()?;
2570        self.persist_candidates().await?;
2571
2572        if resource_blocked {
2573            return self.stop_for_resource();
2574        }
2575        if selected_failed {
2576            self.state.status = RunStatus::Blocked;
2577            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2578            self.state.save()?;
2579            bail!(
2580                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2581            );
2582        }
2583        self.after_implement()
2584    }
2585
2586    /// Retry only persistence after a completed implementation turn.
2587    async fn persist_candidates(&mut self) -> Result<()> {
2588        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2589        for label in labels {
2590            let i = self
2591                .state
2592                .candidates
2593                .iter()
2594                .position(|c| c.label == label)
2595                .context("pending candidate is missing")?;
2596            let c = self.state.candidates[i].clone();
2597            let interrupted = self.state.interrupted_implementations.contains(&label);
2598            let result = async {
2599                let wait = crate::repo_lock::WaitPolicy::new(
2600                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2601                    || false,
2602                );
2603                let _coordination =
2604                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2605                // Best effort evidence requires no index writes. A successful snapshot
2606                // never authorizes removal of a tree whose rescue failed.
2607                if let Err(e) = git::snapshot_uncommitted(
2608                    &c.worktree,
2609                    &self
2610                        .state
2611                        .dir()
2612                        .join("artifacts")
2613                        .join(format!("candidate-{label}-recovery")),
2614                )
2615                .await
2616                {
2617                    self.state.event(
2618                        "preserve",
2619                        format!("candidate {label} snapshot incomplete: {e:#}"),
2620                    );
2621                }
2622                let rescued = git::rescue_commit(
2623                    &c.worktree,
2624                    &format!("magi: candidate {label} (uncommitted work)"),
2625                )
2626                .await?;
2627                self.state.note_withheld("implement", &rescued.withheld);
2628                let base = &self.state.base_commit;
2629                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2630                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2631                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2632                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2633                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2634                if c.empty && commits > 0 && !patch.trim().is_empty()
2635                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2636                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2637                        || self.state.tally.is_some())
2638                {
2639                    // Keep the old classification and pending marker so every
2640                    // retry stops too. Retain decisions as recovery evidence;
2641                    // only a supported recovery operation may replace them.
2642                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2643                }
2644                let c = &mut self.state.candidates[i];
2645                c.commits = commits;
2646                c.stat = stat;
2647                c.files = files;
2648                let no_changes = commits == 0 || patch.trim().is_empty();
2649                c.empty = no_changes && !interrupted;
2650                if interrupted && no_changes {
2651                    c.failed = None;
2652                    c.verified_noop = None;
2653                }
2654                if !c.empty {
2655                    c.failed = None;
2656                    c.verified_noop = None;
2657                }
2658                self.state.event(
2659                    "preserve",
2660                    format!("candidate {label}: {files} files, {commits} commits"),
2661                );
2662                Ok::<_, anyhow::Error>(())
2663            }
2664            .await;
2665            match result {
2666                Ok(()) => {
2667                    self.state.candidate_preservation.remove(&label);
2668                    if !interrupted || self.state.candidates[i].files > 0 {
2669                        self.state.interrupted_implementations.remove(&label);
2670                    }
2671                }
2672                Err(e) => {
2673                    let reason = format!("candidate {label} preservation failed: {e:#}");
2674                    self.state.event("preserve", &reason);
2675                    self.state.candidate_preservation.insert(label, reason);
2676                }
2677            }
2678            if self.state.candidate_preservation.is_empty()
2679                && self.state.status == RunStatus::Blocked
2680                && self.state.candidates.iter().all(|c| c.failed.is_none())
2681            {
2682                // Persist the recovered phase with the cleared pending marker;
2683                // a crash between saves must not turn this into a selected-seat
2684                // failure hold on the next resume.
2685                self.state.status = RunStatus::Implementing;
2686                self.state.event(
2687                    "preserve",
2688                    "candidate persistence recovered; implementation can continue",
2689                );
2690            }
2691            self.state.save()?;
2692        }
2693        if !self.state.candidate_preservation.is_empty() {
2694            self.state.status = RunStatus::Blocked;
2695            self.state.save()?;
2696            bail!(
2697                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2698            );
2699        }
2700        Ok(())
2701    }
2702
2703    /// Ask again, once, for work a CLI did and then failed to hand over.
2704    ///
2705    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2706    /// status with an empty response and a usage report showing output tokens,
2707    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2708    /// seven minutes and 14,267 output tokens that arrived as an empty
2709    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2710    ///
2711    /// Two conditions, and both matter:
2712    ///
2713    /// - **Only when the tree is untouched.** Often the agent has already
2714    ///   written its files and only the closing message was lost; the rescue
2715    ///   commit below picks that up and there is nothing to ask for. Re-asking
2716    ///   then would pay for a second implementation of work already on disk.
2717    /// - **Once.** A CLI that drops one stream can drop the next, and this
2718    ///   node is the most expensive in the graph.
2719    ///
2720    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2721    /// dropped reply still carried its `conversation_id`, so the seat is asked
2722    /// to finish what it was doing rather than sent the whole task again. It
2723    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2724    /// node's - for the same reason a re-ranked judge does: restating finished
2725    /// work is not the work.
2726    ///
2727    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2728    /// same way until it resets, while an abandoned conversation is still
2729    /// there to be picked up.
2730    async fn resume_undelivered(
2731        &mut self,
2732        results: &mut [(usize, SeatState, AgentOutcome)],
2733        sent: &[SeatJob],
2734        prompts: &Prompts,
2735        run_id: &str,
2736    ) {
2737        for (wi, seat, out) in results.iter_mut() {
2738            let Some(dropped) = (match &*out {
2739                AgentOutcome::Dropped(o) => o.dropped.clone(),
2740                _ => None,
2741            }) else {
2742                continue;
2743            };
2744            let Some(job) = sent.get(*wi) else { continue };
2745            // Already on disk? Then only the closing message was lost.
2746            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2747                self.state.event(
2748                    "implement",
2749                    format!(
2750                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2751                         work is in the tree",
2752                        seat.key, dropped.output_tokens, dropped.why
2753                    ),
2754                );
2755                continue;
2756            }
2757            // The re-ask only makes sense as a resume: `resume_after_drop`
2758            // says nothing about the task, trusting the seat to still hold it.
2759            // Without a session to resume — sessions disabled, or this CLI's
2760            // drop shape happened not to carry a session id — that prompt
2761            // would open a brand-new conversation with no context at all,
2762            // which is worse than leaving this as the ordinary failure it
2763            // already is.
2764            if !has_context(&job.spec, seat, job.sessions) {
2765                self.state.event(
2766                    "implement",
2767                    format!(
2768                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2769                         is no session left to resume",
2770                        seat.key, dropped.output_tokens, dropped.why
2771                    ),
2772                );
2773                continue;
2774            }
2775            self.state.event(
2776                "implement",
2777                format!(
2778                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2779                     conversation",
2780                    seat.key, dropped.output_tokens, dropped.why
2781                ),
2782            );
2783            let mut retry = job.clone();
2784            retry.seat = seat.clone();
2785            retry.prompt = prompt::resume_after_drop(&dropped.why);
2786            retry.timeout = retry_budget(job.timeout, true);
2787            retry.stem = format!("{}-resume", job.stem);
2788            let cache = self.state.config.cache_dir();
2789            let ctx = WaveCtx {
2790                carry_seats: false,
2791                pauses: Some((&self.pause, &self.interrupt)),
2792                run: run_id,
2793                node: "implement",
2794                prompts,
2795                cache: cache.as_deref(),
2796                round: None,
2797            };
2798            let (resumed_seat, resumed) =
2799                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2800            *seat = resumed_seat;
2801            *out = resumed;
2802        }
2803    }
2804
2805    /// Fall an implement seat through to the next untried agent in the
2806    /// implementer roster when it lost to quota — or, since the handover was
2807    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2808    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2809    /// quota path itself being unchanged — instead of leaving the
2810    /// seat's loss final the moment one agent's account runs dry.
2811    ///
2812    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2813    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2814    /// truncates to the single slot rotation picked, so a solo task whose one
2815    /// implementer hits quota mid-run used to have nothing else to try. This
2816    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2817    /// unrotated roster — which is the only place the *other* candidates in
2818    /// the machine's roster still exist once `implementers` has been cut down
2819    /// to size.
2820    ///
2821    /// Walks forward from just past the seat's own original position in the
2822    /// roster, never wrapping back to the front: a later candidate slot (say
2823    /// `beta`, the roster's second entry) must fall through to the *next*
2824    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2825    /// almost certainly a different candidate's own agent already — and once
2826    /// the roster's tail is exhausted there is nothing left to fall through
2827    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2828    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2829    /// let this retry that id forever. The loop keeps falling through until
2830    /// an attempt lands something other than `Quota` or the roster's tail
2831    /// runs out of untried ids, at which point the seat is left exactly as
2832    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2833    /// `QuotaLoss` recorded, the candidate failed/empty.
2834    ///
2835    /// `sent` is taken mutably and updated with the fallback agent's spec:
2836    /// `resume_unconfirmed_commands`, which runs after this and also reads
2837    /// `sent`, must see whichever agent actually ended up answering the seat
2838    /// — reading the stale, original spec there would check session
2839    /// eligibility against the wrong CLI and could hand a fallback agent's
2840    /// session id to the agent that just lost the seat to quota.
2841    ///
2842    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2843    /// — `self.seat` only reuses state when the agent id is unchanged, so
2844    /// handing it a different id already gets this for free. Reusing the old
2845    /// seat would resume a different CLI's session as if it were a
2846    /// continuation of this one.
2847    ///
2848    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2849    /// a quota loss cuts an agent off mid-turn, so anything already in the
2850    /// tree is unfinished work, not a completed candidate a re-ask would pay
2851    /// for twice. A dirty tree is rescued into a commit first (the same
2852    /// neutral-identity rescue `implement`'s own outcome loop gives every
2853    /// candidate) so the next agent starts clean.
2854    ///
2855    /// The new agent gets the implementer's full prompt and full
2856    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2857    /// it has no session and no context, and is implementing the task from
2858    /// nothing, unlike a resumed drop which is only restating work already
2859    /// done.
2860    ///
2861    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2862    /// `implement` event, never into `self.state.quota` — that is what
2863    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2864    /// go unspent, and a seat that ultimately recovered on its second or
2865    /// third agent is not the stalled panel that check exists to catch. Only
2866    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2867    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2868    /// outcome loop already has — this helper never pushes to it itself.
2869    async fn resume_seat_handovers(
2870        &mut self,
2871        results: &mut [(usize, SeatState, AgentOutcome)],
2872        sent: &mut [SeatJob],
2873        prompts: &Prompts,
2874        run_id: &str,
2875    ) -> Result<()> {
2876        let instruction = seeded_instruction(&self.state);
2877        let language = self.state.config.graph.language.clone();
2878        let github_language = self.state.config.graph.github_language.clone();
2879        let brief = self
2880            .state
2881            .advice
2882            .as_ref()
2883            .and_then(|a| a.synthesis.as_deref())
2884            .map(str::to_owned);
2885        let attachments = self.state.attachments.clone();
2886        for (wi, seat, out) in results.iter_mut() {
2887            // Who holds the other candidate seats of this wave right now
2888            // (earlier handovers already written back to `sent`).
2889            let others: BTreeSet<String> = sent
2890                .iter()
2891                .enumerate()
2892                .filter(|(j, _)| j != wi)
2893                .map(|(_, j)| j.spec.id.clone())
2894                .collect();
2895            let Some(job) = sent.get_mut(*wi) else {
2896                continue;
2897            };
2898            // Where the seat's own original agent sits in the roster — the
2899            // fallback walk starts just past here, never at the front, so a
2900            // later candidate slot's quota loss does not fall back onto an
2901            // earlier slot's own agent.
2902            let start = self
2903                .roles
2904                .implementer_roster
2905                .iter()
2906                .position(|s| s.id == job.spec.id)
2907                .unwrap_or(0);
2908            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2909            let mut fallback_attempt = 0usize;
2910            let mut prev: Option<FailClass> = None;
2911            while let Some(cur) = FailClass::of(&*out) {
2912                if !should_hand_over(prev.as_ref(), &cur) {
2913                    break;
2914                }
2915                let Some(next) =
2916                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2917                        .cloned()
2918                else {
2919                    break;
2920                };
2921                tried.insert(next.id.clone());
2922                fallback_attempt += 1;
2923
2924                let coordination = self.coordinate(&job.cwd).await?;
2925                if let Ok(r) = git::rescue_commit(
2926                    &job.cwd,
2927                    &format!(
2928                        "magi: candidate {} (uncommitted work before {} fallback)",
2929                        seat.key,
2930                        if cur == FailClass::Quota {
2931                            "quota"
2932                        } else {
2933                            "handover"
2934                        }
2935                    ),
2936                )
2937                .await
2938                {
2939                    self.state.note_withheld("implement", &r.withheld);
2940                }
2941                drop(coordination);
2942
2943                record_handover(
2944                    &mut self.state,
2945                    "implement",
2946                    &seat.key,
2947                    &seat.agent,
2948                    &next.id,
2949                    &cur,
2950                    &fail_reason(&*out),
2951                );
2952                prev = Some(cur.clone());
2953
2954                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2955                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2956                // Kept in sync on `sent` itself, not just the local retry: a
2957                // later helper (`resume_unconfirmed_commands`) reads `sent`
2958                // after this one returns and must see whichever agent is now
2959                // occupying the seat, not the one that just quota'd out —
2960                // otherwise it would judge session/continuation eligibility
2961                // by the wrong CLI and could resend a fallback's session id
2962                // to the agent that lost it the seat in the first place.
2963                job.spec = next.clone();
2964                let mut retry = job.clone();
2965                retry.seat = new_seat;
2966                retry.prompt = prompt::implement_in(
2967                    &instruction,
2968                    &job.cwd.to_string_lossy(),
2969                    &language,
2970                    &github_language,
2971                    brief.as_deref(),
2972                    &attachments,
2973                );
2974                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2975                let cache = self.state.config.cache_dir();
2976                let ctx = WaveCtx {
2977                    carry_seats: false,
2978                    pauses: Some((&self.pause, &self.interrupt)),
2979                    run: run_id,
2980                    node: "implement",
2981                    prompts,
2982                    cache: cache.as_deref(),
2983                    round: None,
2984                };
2985                let (fallback_seat, fallback_out) = run_one(
2986                    retry,
2987                    Arc::clone(&self.sem),
2988                    &ctx,
2989                    &mut self.state,
2990                    fallback_attempt,
2991                )
2992                .await;
2993                *seat = fallback_seat;
2994                *out = fallback_out;
2995            }
2996        }
2997        Ok(())
2998    }
2999
3000    /// Ask an implement seat's own CLI to confirm what it started, once, when
3001    /// its reply reported a command whose completion status it never
3002    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3003    /// that does and does not mean.
3004    ///
3005    /// The completion contract this task asks for, extended to `implement`
3006    /// with the same signal `continue_fix_report` reads for the fixer,
3007    /// rather than a keyword search over the reply or a hard requirement on
3008    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3009    /// a candidate's CLI turn ended cleanly while a test run it had started
3010    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3011    /// named in it at all is untouched by this: `commands` is empty, so
3012    /// there is nothing to be unconfirmed.
3013    ///
3014    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3015    /// this is not about recovering edits that might already be on disk, it
3016    /// is about a result the seat itself never vouched for, which resuming
3017    /// asks for regardless of what the tree already holds. Bounded to one
3018    /// attempt for the same reason `resume_undelivered` is — this is the
3019    /// most expensive node in the graph — and a seat that still cannot
3020    /// confirm on that attempt is left as whatever its (possibly still
3021    /// unconfirmed) reply says; this does not invent a new "failed" reason
3022    /// for a candidate that otherwise produced a real, committed change.
3023    async fn resume_unconfirmed_commands(
3024        &mut self,
3025        results: &mut [(usize, SeatState, AgentOutcome)],
3026        sent: &[SeatJob],
3027        prompts: &Prompts,
3028        run_id: &str,
3029    ) {
3030        for (wi, seat, out) in results.iter_mut() {
3031            let AgentOutcome::Ok(o) = &*out else {
3032                continue;
3033            };
3034            if !has_unconfirmed_command(&o.commands) {
3035                continue;
3036            }
3037            let Some(job) = sent.get(*wi) else { continue };
3038            if !has_context(&job.spec, seat, job.sessions) {
3039                self.state.event(
3040                    "implement",
3041                    format!(
3042                        "{}: the reply named a command whose own CLI never confirmed the exit \
3043                         status of, but there is no session left to resume",
3044                        seat.key
3045                    ),
3046                );
3047                continue;
3048            }
3049            self.state.event(
3050                "implement",
3051                format!(
3052                    "{}: the reply named a command whose own CLI never confirmed the exit \
3053                     status of; resuming the conversation",
3054                    seat.key
3055                ),
3056            );
3057            let mut retry = job.clone();
3058            retry.seat = seat.clone();
3059            retry.prompt = prompt::resume_incomplete(
3060                "a command in your last reply had no confirmed exit status",
3061            );
3062            retry.timeout = retry_budget(job.timeout, true);
3063            retry.stem = format!("{}-confirm", job.stem);
3064            let cache = self.state.config.cache_dir();
3065            let ctx = WaveCtx {
3066                carry_seats: false,
3067                pauses: Some((&self.pause, &self.interrupt)),
3068                run: run_id,
3069                node: "implement",
3070                prompts,
3071                cache: cache.as_deref(),
3072                round: None,
3073            };
3074            let (resumed_seat, resumed) =
3075                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3076            *seat = resumed_seat;
3077            *out = resumed;
3078        }
3079    }
3080
3081    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3082    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3083    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3084    /// that motivated this.
3085    ///
3086    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3087    /// own nudge loop already covers for judge/review/vote seats, and not a
3088    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3089    /// implement seats: here the CLI turn genuinely finished while the node's
3090    /// own work — the fixer's account of what it did — had not. Gated purely
3091    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3092    /// reply, never on any wording in it, so a fixer whose valid, first-try
3093    /// `FixReport` happens to mention having waited on a background test is
3094    /// never resumed — the `Ok(report)` branch at the call site returns
3095    /// before this is ever invoked.
3096    ///
3097    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3098    /// attempt ([`retry_budget`]), nothing attempted once the session is
3099    /// gone, and a quota hit ends the loop immediately rather than retrying a
3100    /// rate limit that fails the same way again.
3101    async fn continue_fix_report(
3102        &mut self,
3103        mut seat: SeatState,
3104        parse_err: String,
3105        job: &SeatJob,
3106        prompts: &Prompts,
3107        run_id: &str,
3108        round: usize,
3109    ) -> (
3110        SeatState,
3111        Option<FixReport>,
3112        Option<String>,
3113        ContinuationRecord,
3114    ) {
3115        let mut last_err = parse_err;
3116        let mut cumulative_wait_ms = 0u64;
3117        let mut attempts = 0usize;
3118        loop {
3119            if !has_context(&job.spec, &seat, job.sessions) {
3120                self.state.event(
3121                    "fix",
3122                    format!(
3123                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3124                         session left to resume into"
3125                    ),
3126                );
3127                let outcome = if attempts == 0 {
3128                    ContinuationOutcome::NoSession
3129                } else {
3130                    ContinuationOutcome::Exhausted
3131                };
3132                return (
3133                    seat,
3134                    None,
3135                    Some(format!("unparsable fix report: {last_err}")),
3136                    ContinuationRecord {
3137                        attempts,
3138                        cumulative_wait_ms,
3139                        outcome,
3140                    },
3141                );
3142            }
3143            if attempts >= MAX_FIX_CONTINUATIONS {
3144                self.state.event(
3145                    "fix",
3146                    format!(
3147                        "round {round}: fixer's reply still had no adoption report after \
3148                         {attempts} continuation(s) ({last_err}); giving up"
3149                    ),
3150                );
3151                return (
3152                    seat,
3153                    None,
3154                    Some(format!(
3155                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3156                    )),
3157                    ContinuationRecord {
3158                        attempts,
3159                        cumulative_wait_ms,
3160                        outcome: ContinuationOutcome::Exhausted,
3161                    },
3162                );
3163            }
3164            attempts += 1;
3165            self.state.event(
3166                "fix",
3167                format!(
3168                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3169                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3170                ),
3171            );
3172            let mut retry = job.clone();
3173            retry.seat = seat.clone();
3174            retry.prompt = prompt::resume_incomplete(&last_err);
3175            retry.timeout = retry_budget(job.timeout, true);
3176            retry.stem = format!("{}-continue{attempts}", job.stem);
3177            let cache = self.state.config.cache_dir();
3178            let ctx = WaveCtx {
3179                carry_seats: false,
3180                pauses: Some((&self.pause, &self.interrupt)),
3181                run: run_id,
3182                node: "fix",
3183                prompts,
3184                cache: cache.as_deref(),
3185                round: Some(round),
3186            };
3187            let (resumed_seat, resumed_out) = run_one(
3188                retry,
3189                Arc::clone(&self.sem),
3190                &ctx,
3191                &mut self.state,
3192                attempts,
3193            )
3194            .await;
3195            seat = resumed_seat;
3196            match resumed_out {
3197                AgentOutcome::Ok(o) => {
3198                    cumulative_wait_ms += o.duration_ms;
3199                    match verdict::extract_json::<FixReport>(&o.text) {
3200                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3201                            self.state.event(
3202                                "fix",
3203                                format!(
3204                                    "round {round}: fixer's adoption report recovered after \
3205                                     {attempts} continuation(s)"
3206                                ),
3207                            );
3208                            return (
3209                                seat,
3210                                Some(report),
3211                                None,
3212                                ContinuationRecord {
3213                                    attempts,
3214                                    cumulative_wait_ms,
3215                                    outcome: ContinuationOutcome::Resumed,
3216                                },
3217                            );
3218                        }
3219                        // The report parsed, but this same reply's own
3220                        // CommandEvidence — the identical record `state.jobs`
3221                        // renders — names a command whose CLI never
3222                        // confirmed an exit status. Read together, that is
3223                        // not a resolved answer: keep nudging rather than
3224                        // accept a report standing next to a command the
3225                        // seat's own CLI cannot vouch for.
3226                        Ok(_) => {
3227                            last_err = "the reply parsed, but it reported a command whose own CLI \
3228                                 never confirmed an exit status"
3229                                .to_owned();
3230                        }
3231                        Err(e) => last_err = e.to_string(),
3232                    }
3233                }
3234                AgentOutcome::Quota(o) => {
3235                    cumulative_wait_ms += o.duration_ms;
3236                    self.state.quota.push(QuotaLoss {
3237                        seat: seat.key.clone(),
3238                        node: "fix".to_owned(),
3239                        at: Timestamp::now(),
3240                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3241                    });
3242                    self.state.event(
3243                        "fix",
3244                        format!(
3245                            "round {round}: continuation rate limited (quota); not retrying now"
3246                        ),
3247                    );
3248                    return (
3249                        seat,
3250                        None,
3251                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3252                        ContinuationRecord {
3253                            attempts,
3254                            cumulative_wait_ms,
3255                            outcome: ContinuationOutcome::QuotaLost,
3256                        },
3257                    );
3258                }
3259                AgentOutcome::Dropped(o) => {
3260                    cumulative_wait_ms += o.duration_ms;
3261                    let why = o
3262                        .dropped
3263                        .as_ref()
3264                        .map(|d| d.why.as_str())
3265                        .unwrap_or("the CLI ended the stream without delivering its answer");
3266                    last_err = format!("the CLI dropped the stream ({why})");
3267                }
3268                AgentOutcome::Failed(e) => last_err = e,
3269                // The continuation was never started; the report stays lost
3270                // and the caller records that, without any handover.
3271                AgentOutcome::ResourceBlocked(e) => {
3272                    return (
3273                        seat,
3274                        None,
3275                        Some(format!("the continuation was not started: {e}")),
3276                        ContinuationRecord {
3277                            attempts: attempts.saturating_sub(1),
3278                            cumulative_wait_ms,
3279                            outcome: ContinuationOutcome::NoSession,
3280                        },
3281                    );
3282                }
3283            }
3284        }
3285    }
3286
3287    fn after_implement(&mut self) -> Result<()> {
3288        if !self.state.candidate_preservation.is_empty() {
3289            bail!("candidate preservation incomplete; judging is unsafe");
3290        }
3291        // Scan every candidate patch once the set is complete.
3292        if self.state.leaks.is_empty() {
3293            let cfg = self.state.config.blind.clone();
3294            let mut leaks = Vec::new();
3295            for c in &self.state.candidates {
3296                let Some(patch) =
3297                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3298                else {
3299                    continue;
3300                };
3301                leaks.extend(blind::scan(
3302                    &format!("candidate {} patch", c.label),
3303                    &patch,
3304                    &cfg.vendor_tokens,
3305                ));
3306            }
3307            if !leaks.is_empty() {
3308                let summary = leaks
3309                    .iter()
3310                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3311                    .collect::<Vec<_>>()
3312                    .join(", ");
3313                match cfg.on_leak {
3314                    LeakPolicy::Fail => {
3315                        self.state.status = RunStatus::Failed;
3316                        self.state
3317                            .event("blind", format!("vendor text in a patch: {summary}"));
3318                        self.state.leaks = leaks;
3319                        self.state.save()?;
3320                        self.settle_questions();
3321                        bail!(
3322                            "blind.on_leak = \"fail\" and vendor text reached a \
3323                             judged patch: {summary}"
3324                        );
3325                    }
3326                    LeakPolicy::Redact => self.state.event(
3327                        "blind",
3328                        format!("redacting vendor text for judging: {summary}"),
3329                    ),
3330                    LeakPolicy::Warn => self.state.event(
3331                        "blind",
3332                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3333                    ),
3334                }
3335                self.state.leaks = leaks;
3336            }
3337        }
3338
3339        if self.state.viable().is_empty() {
3340            if self.state.all_candidates_verified_noop() {
3341                // Every candidate agreed, with evidence the adoption guard
3342                // accepted, that nothing belongs in this worktree. That is
3343                // not the same fact as a candidate that simply failed to
3344                // write anything, and settling it as an ordinary `Failed`
3345                // (see `SCHEMA`'s doc for schema 10) is what let two of
3346                // task 391f's attempts burn a retry each re-discovering the
3347                // same already-landed fix. Terminal either way, so `judge`
3348                // must never run over an empty candidate set — unlike the
3349                // `Failed` branch below this returns `Ok`, not an error:
3350                // nothing here failed.
3351                self.state.status = RunStatus::VerifiedNoop;
3352                self.state.save()?;
3353                self.settle_questions();
3354                return Ok(());
3355            }
3356            self.state.status = RunStatus::Failed;
3357            self.state.save()?;
3358            self.settle_questions();
3359            bail!("no candidate produced a change; nothing to judge");
3360        }
3361        self.state.status = RunStatus::Judging;
3362        self.state.save()?;
3363        Ok(())
3364    }
3365
3366    // --------------------------------------------------------------- judge
3367
3368    async fn judge(&mut self) -> Result<()> {
3369        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3370        // agent files with `magi task add` name the run that paid for it. The
3371        // prompt overlay is cloned alongside it because the waves borrow it
3372        // while `self` is mutably borrowed by the node's own bookkeeping.
3373        let run_id = self.state.id.clone();
3374        let prompts = self.state.config.prompts.clone();
3375        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3376            return Ok(());
3377        }
3378        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3379        if viable.len() == 1 {
3380            // Recorded so this is a one-time event: `judgements` stays empty
3381            // either way, which without this flag is indistinguishable from
3382            // "not yet judged" on the next reentry — and status is left
3383            // untouched, so a later node's conclusion (e.g. `Blocked` after
3384            // the review budget ran out) survives a resume instead of being
3385            // clobbered back to `Judging` by this node running again.
3386            self.state.judge_skipped = true;
3387            self.state.event(
3388                "judge",
3389                format!(
3390                    "only candidate {} produced a change; judging skipped",
3391                    viable[0].label
3392                ),
3393            );
3394            self.state.save()?;
3395            return Ok(());
3396        }
3397        self.state.status = RunStatus::Judging;
3398
3399        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3400        let language = self.state.config.graph.language.clone();
3401        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3402        let sessions = self.state.config.graph.sessions;
3403        let artifacts = agent::artifacts_dir(&self.state.dir());
3404        let root = self.state.worktree_root();
3405        let base_short = short(&self.state.base_commit);
3406
3407        let mut jobs = Vec::new();
3408        let mut orders = Vec::new();
3409        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3410            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3411            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3412            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3413            let seat_key = format!("judge-{}", j + 1);
3414            let seat = self.seat(&seat_key, &spec.id);
3415            jobs.push(SeatJob {
3416                prompt: prompt::judge(
3417                    &self.state.instruction,
3418                    &views,
3419                    self.roles.judges.len(),
3420                    &base_short,
3421                    &language,
3422                ),
3423                spec,
3424                seat,
3425                cwd: root.join(format!("judge-{}", j + 1)),
3426                timeout,
3427                allow_write: false,
3428                sessions,
3429                artifacts: artifacts.clone(),
3430                stem: format!("judge-{}", j + 1),
3431                handover: None,
3432            });
3433        }
3434
3435        self.state.event(
3436            "judge",
3437            format!(
3438                "{} judges ranking {} candidates blind",
3439                jobs.len(),
3440                viable.len()
3441            ),
3442        );
3443        let labels_for_check = labels.clone();
3444        let mut quota_losses = Vec::new();
3445        let cache = self.state.config.cache_dir();
3446        let ctx = WaveCtx {
3447            carry_seats: false,
3448            pauses: Some((&self.pause, &self.interrupt)),
3449            run: &run_id,
3450            node: "judge",
3451            prompts: &prompts,
3452            cache: cache.as_deref(),
3453            round: None,
3454        };
3455        let results = ask_json_wave::<Ranking>(
3456            jobs,
3457            Arc::clone(&self.sem),
3458            self.state.config.graph.retries,
3459            &self.roles.judge_roster,
3460            &ctx,
3461            &mut quota_losses,
3462            &mut self.state,
3463            &move |r: &Ranking| r.validate(&labels_for_check),
3464        )
3465        .await;
3466        if self.state.resource_block.is_some() {
3467            return self.stop_for_resource();
3468        }
3469        self.state.quota.extend(quota_losses);
3470
3471        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3472            let agent_id = seat.agent.clone();
3473            self.state.seats.insert(seat.key.clone(), seat);
3474            let mut record = Judgement {
3475                judge: j + 1,
3476                seat: format!("judge-{}", j + 1),
3477                agent: agent_id,
3478                ranking: Vec::new(),
3479                reasons: BTreeMap::new(),
3480                confidence: None,
3481                strengths: BTreeMap::new(),
3482                order: orders[j].clone(),
3483                failed: None,
3484                duration_ms: 0,
3485            };
3486            match res {
3487                Ok((ranking, out)) => {
3488                    record.ranking = ranking.normalized();
3489                    record.reasons = ranking.reasons;
3490                    record.strengths = ranking.strengths;
3491                    record.confidence = ranking.confidence;
3492                    record.duration_ms = out.duration_ms;
3493                    self.state.event(
3494                        "judge",
3495                        format!(
3496                            "judge {} ranked {}",
3497                            j + 1,
3498                            record.ranking.iter().collect::<String>()
3499                        ),
3500                    );
3501                }
3502                Err(e) => {
3503                    record.failed = Some(e.to_string());
3504                    self.state
3505                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3506                }
3507            }
3508            self.state.judgements.push(record);
3509            self.state.save()?;
3510        }
3511        Ok(())
3512    }
3513
3514    // ---------------------------------------------------------- deliberate
3515
3516    async fn deliberate(&mut self) -> Result<()> {
3517        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3518        // agent files with `magi task add` name the run that paid for it. The
3519        // prompt overlay is cloned alongside it because the waves borrow it
3520        // while `self` is mutably borrowed by the node's own bookkeeping.
3521        let run_id = self.state.id.clone();
3522        let prompts = self.state.config.prompts.clone();
3523        if !self.state.deliberation.is_empty() {
3524            return Ok(());
3525        }
3526        let tops: Vec<char> = self
3527            .state
3528            .judgements
3529            .iter()
3530            .filter_map(|j| j.ranking.first().copied())
3531            .collect();
3532        let rounds = self.state.config.graph.deliberate_rounds;
3533        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3534            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3535                self.state.event(
3536                    "deliberate",
3537                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3538                );
3539            }
3540            self.state.status = RunStatus::Voting;
3541            self.state.save()?;
3542            return Ok(());
3543        }
3544
3545        self.state.status = RunStatus::Deliberating;
3546        self.state.event(
3547            "deliberate",
3548            format!(
3549                "split: first choices were {} — opening {rounds} round(s)",
3550                tops.iter().collect::<String>()
3551            ),
3552        );
3553
3554        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3555        let language = self.state.config.graph.language.clone();
3556        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3557        let sessions = self.state.config.graph.sessions;
3558        let artifacts = agent::artifacts_dir(&self.state.dir());
3559        let root = self.state.worktree_root();
3560        let base_short = short(&self.state.base_commit);
3561
3562        // Judges argue in sequence so that a turn can answer the one before it;
3563        // that is the difference between deliberation and three parallel
3564        // monologues.
3565        for round in 1..=rounds {
3566            let mut turns: Vec<DeliberationTurn> = Vec::new();
3567            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3568                if self.state.judgements[j].failed.is_some() {
3569                    continue;
3570                }
3571                let seat_key = format!("judge-{}", j + 1);
3572                let spec = self.occupant(&seat_key, spec);
3573                let mut seat = self.seat(&seat_key, &spec.id);
3574                let transcript = self.transcript(&turns, j);
3575                let build = |context: Option<&str>| {
3576                    prompt::deliberate(
3577                        &self.state.instruction,
3578                        context,
3579                        &transcript,
3580                        round,
3581                        rounds,
3582                        &language,
3583                    )
3584                };
3585                let block = self.candidate_block(&viable, &base_short);
3586                let full = build(Some(&block));
3587                let text = if has_context(&spec, &seat, sessions) {
3588                    build(None)
3589                } else {
3590                    full.clone()
3591                };
3592                let job = SeatJob {
3593                    spec,
3594                    seat: seat.clone(),
3595                    prompt: text,
3596                    cwd: root.join(format!("judge-{}", j + 1)),
3597                    timeout,
3598                    allow_write: false,
3599                    sessions,
3600                    artifacts: artifacts.clone(),
3601                    stem: format!("delib-{round}-judge-{}", j + 1),
3602                    handover: Some(full),
3603                };
3604                let cache = self.state.config.cache_dir();
3605                let ctx = WaveCtx {
3606                    carry_seats: false,
3607                    pauses: Some((&self.pause, &self.interrupt)),
3608                    run: &run_id,
3609                    node: "deliberate",
3610                    prompts: &prompts,
3611                    cache: cache.as_deref(),
3612                    round: None,
3613                };
3614                // A turn is never nudged (`retries` 0): a failed seat is
3615                // handed to the next roster agent, which gets the full
3616                // context. An empty answer is a turn, not a failure.
3617                let mut losses = Vec::new();
3618                let mut results = ask_wave_with::<String>(
3619                    vec![job],
3620                    Arc::clone(&self.sem),
3621                    0,
3622                    &self.roles.judge_roster,
3623                    &ctx,
3624                    &mut losses,
3625                    &mut self.state,
3626                    &|text: &str| {
3627                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3628                    },
3629                )
3630                .await;
3631                self.state.quota.extend(losses);
3632                let (updated, res, _) = results.pop().expect("one job in, one result out");
3633                seat = updated;
3634                let agent_id = seat.agent.clone();
3635                self.state.seats.insert(seat.key.clone(), seat);
3636                let body = match res {
3637                    Ok((body, _)) => body,
3638                    // Skip the seat; a CLI's raw error JSON is never read as
3639                    // this judge's position.
3640                    Err(e) => {
3641                        self.state
3642                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3643                        continue;
3644                    }
3645                };
3646                let tentative = verdict::extract_json::<Position>(&body)
3647                    .ok()
3648                    .and_then(|p| p.tentative)
3649                    .and_then(|s| s.trim().chars().next())
3650                    .map(|c| c.to_ascii_uppercase());
3651                self.state.event(
3652                    "deliberate",
3653                    format!(
3654                        "round {round}: judge {} now favours {}",
3655                        j + 1,
3656                        tentative.map_or("—".to_owned(), |c| c.to_string())
3657                    ),
3658                );
3659                turns.push(DeliberationTurn {
3660                    judge: j + 1,
3661                    agent: agent_id,
3662                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3663                    tentative,
3664                });
3665            }
3666            self.state
3667                .deliberation
3668                .push(DeliberationRound { round, turns });
3669            self.state.save()?;
3670        }
3671
3672        self.state.status = RunStatus::Voting;
3673        self.state.save()?;
3674        Ok(())
3675    }
3676
3677    // ---------------------------------------------------------------- vote
3678
3679    async fn vote(&mut self) -> Result<()> {
3680        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3681        // agent files with `magi task add` name the run that paid for it. The
3682        // prompt overlay is cloned alongside it because the waves borrow it
3683        // while `self` is mutably borrowed by the node's own bookkeeping.
3684        let run_id = self.state.id.clone();
3685        let prompts = self.state.config.prompts.clone();
3686        if !self.state.votes.is_empty() {
3687            return Ok(());
3688        }
3689        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3690        if viable.len() == 1 {
3691            return Ok(());
3692        }
3693        self.state.status = RunStatus::Voting;
3694
3695        let language = self.state.config.graph.language.clone();
3696        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3697        let sessions = self.state.config.graph.sessions;
3698        let artifacts = agent::artifacts_dir(&self.state.dir());
3699        let root = self.state.worktree_root();
3700        let base_short = short(&self.state.base_commit);
3701        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3702
3703        let mut jobs = Vec::new();
3704        let mut seats_at = Vec::new();
3705        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3706            if self
3707                .state
3708                .judgements
3709                .get(j)
3710                .is_some_and(|r| r.failed.is_some())
3711            {
3712                continue;
3713            }
3714            let seat_key = format!("judge-{}", j + 1);
3715            let spec = self.occupant(&seat_key, spec);
3716            let seat = self.seat(&seat_key, &spec.id);
3717            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3718            let text = if has_context(&spec, &seat, sessions) {
3719                prompt::final_vote(&viable, &language)
3720            } else {
3721                full.clone()
3722            };
3723            jobs.push(SeatJob {
3724                spec,
3725                seat,
3726                prompt: text,
3727                cwd: root.join(format!("judge-{}", j + 1)),
3728                timeout,
3729                allow_write: false,
3730                sessions,
3731                artifacts: artifacts.clone(),
3732                stem: format!("vote-judge-{}", j + 1),
3733                handover: Some(full),
3734            });
3735            seats_at.push(j);
3736        }
3737
3738        self.state.event(
3739            "vote",
3740            format!(
3741                "collecting {} final votes one by one, privately",
3742                jobs.len()
3743            ),
3744        );
3745        let allowed = viable.clone();
3746        let mut quota_losses = Vec::new();
3747        let cache = self.state.config.cache_dir();
3748        let ctx = WaveCtx {
3749            carry_seats: false,
3750            pauses: Some((&self.pause, &self.interrupt)),
3751            run: &run_id,
3752            node: "vote",
3753            prompts: &prompts,
3754            cache: cache.as_deref(),
3755            round: None,
3756        };
3757        let results = ask_json_wave::<FinalVote>(
3758            jobs,
3759            Arc::clone(&self.sem),
3760            self.state.config.graph.retries,
3761            &self.roles.judge_roster,
3762            &ctx,
3763            &mut quota_losses,
3764            &mut self.state,
3765            &move |v: &FinalVote| match v.label() {
3766                Some(c) if allowed.contains(&c) => Ok(()),
3767                other => bail!("vote {other:?} is not one of {allowed:?}"),
3768            },
3769        )
3770        .await;
3771        if self.state.resource_block.is_some() {
3772            return self.stop_for_resource();
3773        }
3774        self.state.quota.extend(quota_losses);
3775
3776        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3777            let agent_id = seat.agent.clone();
3778            self.state.seats.insert(seat.key.clone(), seat);
3779            let initial = self
3780                .state
3781                .judgements
3782                .get(j)
3783                .and_then(|r| r.ranking.first().copied());
3784            let mut record = VoteRecord {
3785                judge: j + 1,
3786                agent: agent_id,
3787                vote: None,
3788                reason: String::new(),
3789                changed: false,
3790            };
3791            match res {
3792                Ok((v, _)) => {
3793                    record.vote = v.label();
3794                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3795                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3796                    self.state.event(
3797                        "vote",
3798                        format!(
3799                            "judge {} voted {}{}",
3800                            j + 1,
3801                            record.vote.unwrap_or('?'),
3802                            if record.changed { " (changed)" } else { "" }
3803                        ),
3804                    );
3805                }
3806                Err(e) => {
3807                    self.state
3808                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3809                }
3810            }
3811            self.state.votes.push(record);
3812            self.state.save()?;
3813        }
3814        Ok(())
3815    }
3816
3817    // --------------------------------------------------------------- tally
3818
3819    fn tally(&mut self) -> Result<()> {
3820        if self.state.tally.is_some() {
3821            return Ok(());
3822        }
3823        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3824        let tops: Vec<char> = self
3825            .state
3826            .judgements
3827            .iter()
3828            .filter_map(|j| j.ranking.first().copied())
3829            .collect();
3830        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3831
3832        // A judge whose private vote failed still counted once, in the initial
3833        // ranking; using it beats discarding a whole seat.
3834        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3835        let mut cast: Vec<char> = Vec::new();
3836        for (i, j) in self.state.judgements.iter().enumerate() {
3837            let vote = self
3838                .state
3839                .votes
3840                .iter()
3841                .find(|v| v.judge == i + 1)
3842                .and_then(|v| v.vote)
3843                .or_else(|| j.ranking.first().copied());
3844            if let Some(v) = vote {
3845                *first_choice.entry(v).or_insert(0) += 1;
3846                cast.push(v);
3847            }
3848        }
3849
3850        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3851        for j in &self.state.judgements {
3852            let n = j.ranking.len();
3853            for (pos, label) in j.ranking.iter().enumerate() {
3854                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3855            }
3856        }
3857
3858        let best = first_choice.values().copied().max().unwrap_or(0);
3859        let mut leaders: Vec<char> = first_choice
3860            .iter()
3861            .filter(|(_, v)| **v == best)
3862            .map(|(k, _)| *k)
3863            .collect();
3864        let mut tie_break = None;
3865        if leaders.len() > 1 {
3866            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3867            let borda_leaders: Vec<char> = leaders
3868                .iter()
3869                .copied()
3870                .filter(|l| borda[l] == top_borda)
3871                .collect();
3872            tie_break = Some(if borda_leaders.len() == 1 {
3873                format!(
3874                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3875                    leaders.len()
3876                )
3877            } else {
3878                format!(
3879                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3880                    leaders.len()
3881                )
3882            });
3883            leaders = borda_leaders;
3884            leaders.sort_unstable();
3885        }
3886        let winner = *leaders
3887            .first()
3888            .or(viable.first())
3889            .context("no candidate to declare a winner from")?;
3890
3891        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3892        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3893        let deliberated = !self.state.deliberation.is_empty();
3894
3895        // Whose verdict is this? A rate-limited seat is absent even if it
3896        // ranked before the limit hit, so presence is measured against the
3897        // recorded losses, not just "did a ranking ever appear".
3898        let quota_seats: std::collections::BTreeSet<&str> =
3899            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3900        let mut present = 0usize;
3901        for (i, j) in self.state.judgements.iter().enumerate() {
3902            if quota_seats.contains(j.seat.as_str()) {
3903                continue;
3904            }
3905            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3906            let voted = self
3907                .state
3908                .votes
3909                .iter()
3910                .any(|v| v.judge == i + 1 && v.vote.is_some());
3911            if ranked || voted {
3912                present += 1;
3913            }
3914        }
3915        // Strict majority of the configured panel. A bare majority is real
3916        // signal we can act on, while a minority verdict must never stand in
3917        // for a healthy one. A one-candidate run needs no panel at all, and
3918        // `judges` stays `0` rather than the roster size a panel that never
3919        // sat would otherwise be credited with.
3920        let needs_quorum = viable.len() > 1;
3921        let judges_total = if needs_quorum {
3922            self.roles.judges.len()
3923        } else {
3924            0
3925        };
3926        let quorum = if needs_quorum {
3927            judges_total / 2 + 1
3928        } else {
3929            0
3930        };
3931        let met_quorum = !needs_quorum || present >= quorum;
3932        let uncontested = (!needs_quorum).then(|| {
3933            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3934        });
3935
3936        self.state.event(
3937            "tally",
3938            match &uncontested {
3939                Some(reason) => format!("winner {winner} — {reason}"),
3940                None => format!(
3941                    "winner {winner} — votes {} | initial {} | {} changed | \
3942                     {present}/{judges_total} judges{}",
3943                    first_choice
3944                        .iter()
3945                        .map(|(k, v)| format!("{k}:{v}"))
3946                        .collect::<Vec<_>>()
3947                        .join(" "),
3948                    if unanimous_initial {
3949                        "unanimous"
3950                    } else {
3951                        "split"
3952                    },
3953                    changed_votes,
3954                    if met_quorum {
3955                        String::new()
3956                    } else {
3957                        format!(" — below quorum ({quorum} required)")
3958                    },
3959                ),
3960            },
3961        );
3962        if !met_quorum {
3963            self.state.event(
3964                "stall",
3965                format!(
3966                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3967                     the run stops here, resumable"
3968                ),
3969            );
3970        }
3971        self.state.tally = Some(Tally {
3972            first_choice,
3973            borda,
3974            winner,
3975            rankings: tops.len(),
3976            unanimous_initial,
3977            deliberated,
3978            changed_votes,
3979            unanimous_final,
3980            tie_break,
3981            judges: judges_total,
3982            present,
3983            quorum,
3984            met_quorum,
3985            uncontested,
3986        });
3987        self.state.status = if met_quorum {
3988            RunStatus::Reviewing
3989        } else {
3990            RunStatus::Stalled
3991        };
3992        self.state.save()?;
3993        Ok(())
3994    }
3995
3996    // ------------------------------------------------------------- recover
3997
3998    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3999    /// resumed toward completion once the transient cause clears.
4000    ///
4001    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4002    /// it toward the quorum, which is exactly the set of seats whose absence
4003    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4004    /// not depend on which node happened to hit the limit), or an ordinary
4005    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4006    /// seat is never disturbed.
4007    ///
4008    /// A seat that now answers with a usable ranking is "recovered": its
4009    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4010    /// `tally` counts it present again), and its vote re-collected. A seat that
4011    /// still fails keeps its loss and stays absent.
4012    ///
4013    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4014    /// to review/gate/merge), `false` when it is still below quorum (the run
4015    /// stays `Stalled`, still resumable for a later retry).
4016    #[allow(clippy::too_many_lines)]
4017    async fn recover_stall(&mut self) -> Result<bool> {
4018        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4019        // agent files with `magi task add` name the run that paid for it. The
4020        // prompt overlay is cloned alongside it because the waves borrow it
4021        // while `self` is mutably borrowed by the node's own bookkeeping.
4022        let run_id = self.state.id.clone();
4023        let prompts = self.state.config.prompts.clone();
4024        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4025        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4026        // is what keeps a non-quota collapse — or a quota loss recorded at the
4027        // deliberate node — from being a permanent dead-end on `--resume`.
4028        let quota_seats: BTreeSet<&str> =
4029            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4030        let absent: Vec<String> = self
4031            .state
4032            .judgements
4033            .iter()
4034            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4035            .map(|j| j.seat.clone())
4036            .collect();
4037        if absent.is_empty() {
4038            return Ok(false);
4039        }
4040        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4041        if viable.len() <= 1 {
4042            return Ok(false);
4043        }
4044        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4045        let language = self.state.config.graph.language.clone();
4046        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4047        let sessions = self.state.config.graph.sessions;
4048        let artifacts = agent::artifacts_dir(&self.state.dir());
4049        let root = self.state.worktree_root();
4050        let base_short = short(&self.state.base_commit);
4051        let candidates: Vec<Candidate> = viable.clone();
4052
4053        // Map each absent seat key to its 0-based position in `roles.judges`.
4054        let mut positions: Vec<usize> = absent
4055            .iter()
4056            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4057            .collect();
4058        if positions.is_empty() {
4059            return Ok(false);
4060        }
4061        positions.sort_unstable();
4062        positions.dedup();
4063
4064        // Re-rank the lost seats, one blind prompt each.
4065        let mut judge_jobs = Vec::new();
4066        for &j in &positions {
4067            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4068            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4069            let seat_key = format!("judge-{}", j + 1);
4070            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4071            let seat = self.seat(&seat_key, &spec.id);
4072            judge_jobs.push(SeatJob {
4073                spec,
4074                seat,
4075                prompt: prompt::judge(
4076                    &self.state.instruction,
4077                    &views,
4078                    self.roles.judges.len(),
4079                    &base_short,
4080                    &language,
4081                ),
4082                cwd: root.join(seat_key),
4083                timeout,
4084                allow_write: false,
4085                sessions,
4086                artifacts: artifacts.clone(),
4087                stem: format!("judge-{}-recover", j + 1),
4088                handover: None,
4089            });
4090        }
4091
4092        let labels_for_check = labels.clone();
4093        let mut judge_losses = Vec::new();
4094        let retries = self.state.config.graph.retries;
4095        let cache = self.state.config.cache_dir();
4096        let ctx = WaveCtx {
4097            carry_seats: false,
4098            pauses: Some((&self.pause, &self.interrupt)),
4099            run: &run_id,
4100            node: "judge",
4101            prompts: &prompts,
4102            cache: cache.as_deref(),
4103            round: None,
4104        };
4105        let results = ask_json_wave::<Ranking>(
4106            judge_jobs,
4107            Arc::clone(&self.sem),
4108            retries,
4109            &self.roles.judge_roster,
4110            &ctx,
4111            &mut judge_losses,
4112            &mut self.state,
4113            &move |r: &Ranking| r.validate(&labels_for_check),
4114        )
4115        .await;
4116        if self.state.resource_block.is_some() {
4117            self.stop_for_resource()?;
4118            return Ok(false);
4119        }
4120
4121        // Refresh the judgement of every seat that ranked again.
4122        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4123        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4124            let agent_id = seat.agent.clone();
4125            self.state.seats.insert(seat.key.clone(), seat);
4126            let record = &mut self.state.judgements[j];
4127            match res {
4128                Ok((ranking, out)) => {
4129                    record.agent = agent_id;
4130                    record.ranking = ranking.normalized();
4131                    record.reasons = ranking.reasons;
4132                    record.strengths = ranking.strengths;
4133                    record.confidence = ranking.confidence;
4134                    record.failed = None;
4135                    record.duration_ms = out.duration_ms;
4136                    recovered.insert(j);
4137                    self.state.event(
4138                        "recover",
4139                        format!("judge {} ranked again after the limit", j + 1),
4140                    );
4141                }
4142                Err(e) => {
4143                    self.state
4144                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4145                }
4146            }
4147        }
4148
4149        // Re-ask the votes of the seats that recovered a ranking.
4150        let mut vote_jobs = Vec::new();
4151        let mut vote_pos: Vec<usize> = Vec::new();
4152        for &j in &recovered {
4153            let seat_key = format!("judge-{}", j + 1);
4154            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4155            let seat = self.seat(&seat_key, &spec.id);
4156            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4157            let text = if has_context(&spec, &seat, sessions) {
4158                prompt::final_vote(&labels, &language)
4159            } else {
4160                full.clone()
4161            };
4162            vote_jobs.push(SeatJob {
4163                spec,
4164                seat,
4165                prompt: text,
4166                cwd: root.join(seat_key),
4167                timeout,
4168                allow_write: false,
4169                sessions,
4170                artifacts: artifacts.clone(),
4171                stem: format!("vote-judge-{}-recover", j + 1),
4172                handover: Some(full),
4173            });
4174            vote_pos.push(j);
4175        }
4176        let allowed = labels.clone();
4177        let mut vote_losses = Vec::new();
4178        let vote_retries = self.state.config.graph.retries;
4179        let vote_cache = self.state.config.cache_dir();
4180        let ctx = WaveCtx {
4181            carry_seats: false,
4182            pauses: Some((&self.pause, &self.interrupt)),
4183            run: &run_id,
4184            node: "vote",
4185            prompts: &prompts,
4186            cache: vote_cache.as_deref(),
4187            round: None,
4188        };
4189        let votes = ask_json_wave::<FinalVote>(
4190            vote_jobs,
4191            Arc::clone(&self.sem),
4192            vote_retries,
4193            &self.roles.judge_roster,
4194            &ctx,
4195            &mut vote_losses,
4196            &mut self.state,
4197            &move |v: &FinalVote| match v.label() {
4198                Some(c) if allowed.contains(&c) => Ok(()),
4199                other => bail!("vote {other:?} is not one of {allowed:?}"),
4200            },
4201        )
4202        .await;
4203        if self.state.resource_block.is_some() {
4204            self.stop_for_resource()?;
4205            return Ok(false);
4206        }
4207        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4208            let agent_id = seat.agent.clone();
4209            self.state.seats.insert(seat.key.clone(), seat);
4210            match res {
4211                Ok((v, _)) => {
4212                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4213                        rec.vote = v.label();
4214                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4215                    } else {
4216                        self.state.votes.push(VoteRecord {
4217                            judge: j + 1,
4218                            agent: agent_id,
4219                            vote: v.label(),
4220                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4221                            changed: false,
4222                        });
4223                    }
4224                    self.state.event(
4225                        "recover",
4226                        format!("judge {} voted again after the limit", j + 1),
4227                    );
4228                }
4229                Err(e) => {
4230                    self.state
4231                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4232                }
4233            }
4234        }
4235
4236        // A seat that ranked again is present even if its re-vote failed —
4237        // `tally` falls back to the initial ranking's first choice — so clear
4238        // its quota loss. Seats that still fail keep theirs and stay absent.
4239        let recovered_keys: BTreeSet<String> = recovered
4240            .iter()
4241            .map(|&j| format!("judge-{}", j + 1))
4242            .collect();
4243        self.state
4244            .quota
4245            .retain(|q| !recovered_keys.contains(&q.seat));
4246        // A seat that hit the limit again is a fresh loss, not the old one:
4247        // replace the stale entry so the history stays one-per-seat and the
4248        // daemon can tell this attempt's loss from a previous session's.
4249        for loss in judge_losses.into_iter().chain(vote_losses) {
4250            if recovered_keys.contains(&loss.seat) {
4251                continue;
4252            }
4253            self.state.quota.retain(|q| q.seat != loss.seat);
4254            self.state.quota.push(loss);
4255        }
4256
4257        // Recompute the verdict from the refreshed panel.
4258        self.state.tally = None;
4259        self.tally()?;
4260        Ok(self
4261            .state
4262            .tally
4263            .as_ref()
4264            .map(|t| t.met_quorum)
4265            .unwrap_or(false))
4266    }
4267
4268    // ---------------------------------------------------------------- graft
4269
4270    /// Port the strengths the judges attributed to the losing candidates onto
4271    /// the winner, once, between the tally and the review loop.
4272    ///
4273    /// Best-effort by construction: it returns nothing and fails nothing. A
4274    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4275    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4276    /// the run carries on as if the phase were off. See [`graft`] for the
4277    /// decisions and `AGENTS.md` for the invariants.
4278    ///
4279    /// The one exception is a rollback that could not be confirmed: the
4280    /// rejected commit may still be the winner's tip, so the run is blocked
4281    /// (resumable; the record stays `Running`, which makes a resume retry the
4282    /// restore) instead of reviewing, gating and merging it.
4283    async fn graft(&mut self) -> Result<()> {
4284        let Err(e) = self.graft_inner().await else {
4285            return Ok(());
4286        };
4287        if e.is::<crate::repo_lock::ResourceBlocked>() {
4288            return Err(e);
4289        }
4290        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4291            self.state.status = RunStatus::Blocked;
4292            self.state.event("graft", format!("blocked: {e:#}"));
4293            self.state.save()?;
4294            return Err(e);
4295        }
4296        // An error after the `Running` record was saved may have left the
4297        // fixer's work in the tree: the record must not stay unsettled
4298        // (a later resume would reset past the review fixes), so confirm the
4299        // restore now, or block.
4300        if let Some(rec) = self.state.graft.clone()
4301            && !rec.outcome.settled()
4302        {
4303            if let Some(winner) = self.state.winner().cloned() {
4304                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4305                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4306                        return Err(re);
4307                    }
4308                    self.state.status = RunStatus::Blocked;
4309                    self.state.event("graft", format!("blocked: {re:#}"));
4310                    self.state.save()?;
4311                    return Err(re);
4312                }
4313            }
4314            if let Some(g) = self.state.graft.as_mut() {
4315                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4316            }
4317        }
4318        self.state.event("graft", format!("skipped: {e:#}"));
4319        if let Err(e) = self.state.save() {
4320            tracing::warn!("save after graft: {e:#}");
4321        }
4322        Ok(())
4323    }
4324
4325    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4326    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4327    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4328        let _coordination = self.coordinate(wt).await?;
4329        crate::recovery::restore_repair(wt, pre)
4330            .await
4331            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4332    }
4333
4334    /// A skip is a run event, recorded once however often the node is re-entered.
4335    fn graft_skipped(&mut self, why: &str) {
4336        let message = format!("skipped: {why}");
4337        if !self
4338            .state
4339            .events
4340            .iter()
4341            .any(|e| e.node == "graft" && e.message == message)
4342        {
4343            self.state.event("graft", message);
4344        }
4345    }
4346
4347    async fn graft_inner(&mut self) -> Result<()> {
4348        // A round that was started and never finished (crash, kill): the
4349        // fixer may have left anything in the tree, so put the winner back.
4350        if let Some(rec) = self.state.graft.clone()
4351            && !rec.outcome.settled()
4352        {
4353            // Reviews exist only if the graft ended without its record being
4354            // settled (a swallowed error): the tree has moved on since, and
4355            // a reset to `pre` would discard the review fixes.
4356            if self.state.reviews.is_empty()
4357                && let Some(winner) = self.state.winner().cloned()
4358            {
4359                self.graft_restore(&winner.worktree, &rec.pre).await?;
4360            }
4361            if rec.outcome == GraftOutcome::WaitingForRestore {
4362                self.state.graft = None;
4363                self.state.save()?;
4364            } else {
4365                if let Some(g) = self.state.graft.as_mut() {
4366                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4367                }
4368                self.state.event(
4369                    "graft",
4370                    "rolled back: the previous round was interrupted; the winner is untouched",
4371                );
4372                self.state.save()?;
4373                return Ok(());
4374            }
4375        }
4376        let plan = match graft::plan(&self.state) {
4377            Ok(plan) => plan,
4378            Err(skip) => {
4379                if skip != graft::Skip::AlreadyDone {
4380                    self.graft_skipped(skip.reason());
4381                    self.state.save()?;
4382                }
4383                return Ok(());
4384            }
4385        };
4386        let Some(winner) = self.state.winner().cloned() else {
4387            return Ok(());
4388        };
4389        let wt = winner.worktree.clone();
4390        if !git::is_clean(&wt).await? {
4391            self.graft_skipped("the winner's worktree has uncommitted changes");
4392            self.state.save()?;
4393            return Ok(());
4394        }
4395        let pre = git::rev_parse(&wt, "HEAD").await?;
4396        // Saved before the fixer is called: a stop from here on is repaired
4397        // by the branch above on the next entry.
4398        self.state.graft = Some(GraftRecord {
4399            pre: pre.clone(),
4400            post: None,
4401            lines: None,
4402            limit: None,
4403            agent: None,
4404            outcome: GraftOutcome::Running,
4405            requested: graft::requested(&plan),
4406            adoption: None,
4407            adoption_error: None,
4408        });
4409        self.state.event(
4410            "graft",
4411            format!(
4412                "porting strengths of candidate(s) {} onto {}",
4413                plan.strengths
4414                    .iter()
4415                    .map(|(l, _, _)| l.to_string())
4416                    .collect::<Vec<_>>()
4417                    .join(", "),
4418                plan.winner
4419            ),
4420        );
4421        self.state.save()?;
4422
4423        let fix_prompt = prompt::graft(
4424            &self.state.instruction,
4425            plan.winner,
4426            &plan.strengths,
4427            &self.state.config.graph.language,
4428        );
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4433        let (_, seat, out) = self
4434            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4435                SeatJob {
4436                    prompt: fix_prompt.clone(),
4437                    spec,
4438                    seat,
4439                    cwd: wt.clone(),
4440                    timeout,
4441                    allow_write: true,
4442                    sessions,
4443                    artifacts: artifacts.clone(),
4444                    stem: "graft".to_owned(),
4445                    handover: None,
4446                }
4447            })
4448            .await;
4449        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4450            // Preserve the need to restore earlier fallback edits without
4451            // confusing an unstarted seat with an interrupted invocation.
4452            self.state.graft.as_mut().expect("graft record").outcome =
4453                GraftOutcome::WaitingForRestore;
4454            return self.stop_for_resource();
4455        }
4456        let mut why: Option<String> = None;
4457        match &out {
4458            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4459                why = Some("the fixer produced nothing usable".to_owned());
4460            }
4461            AgentOutcome::Ok(_) => {}
4462            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4463            AgentOutcome::Quota(o) => {
4464                self.state.quota.push(QuotaLoss {
4465                    seat: seat.key.clone(),
4466                    node: "graft".to_owned(),
4467                    at: Timestamp::now(),
4468                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4469                });
4470                why = Some("rate limited (quota)".to_owned());
4471            }
4472            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4473                why = Some(format!(
4474                    "the fixer failed: {}",
4475                    e.lines()
4476                        .next()
4477                        .unwrap_or("")
4478                        .chars()
4479                        .take(160)
4480                        .collect::<String>()
4481                ));
4482            }
4483        }
4484        let agent_id = seat.agent.clone();
4485        self.state.seats.insert(seat.key.clone(), seat);
4486
4487        let mut post = None;
4488        let mut lines = None;
4489        let _coordination = self.coordinate(&wt).await?;
4490        if why.is_none() {
4491            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4492            if let Ok(r) = &rescued {
4493                self.state.note_withheld("graft", &r.withheld);
4494            }
4495            let head = git::rev_parse(&wt, "HEAD").await?;
4496            // What is measured must be what gets reviewed and gated: a failed
4497            // rescue, or anything still uncommitted (withheld lockfiles
4498            // aside, which are never part of the tree), rejects the graft.
4499            let dirty = match &rescued {
4500                Ok(r) => git::status_porcelain(&wt)
4501                    .await
4502                    .map(|st| {
4503                        st.lines().any(|l| {
4504                            !r.withheld
4505                                .iter()
4506                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4507                        })
4508                    })
4509                    .unwrap_or(true),
4510                Err(_) => true,
4511            };
4512            if dirty {
4513                why = Some("the fixer left changes that could not be committed".to_owned());
4514            } else if head == pre {
4515                why = Some("the fixer changed nothing".to_owned());
4516            } else {
4517                lines = git::changed_lines(&wt, &pre, &head).await;
4518                post = Some(head);
4519            }
4520        }
4521        let (adoption, adoption_error) = match &out {
4522            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4523            _ => (None, None),
4524        };
4525        let requested = graft::requested(&plan);
4526        let outcome = match why {
4527            None => {
4528                self.state.event(
4529                    "graft",
4530                    format!(
4531                        "applied: {}",
4532                        lines.map_or_else(
4533                            || "changed lines unavailable".to_owned(),
4534                            |n| format!("{n} changed lines")
4535                        )
4536                    ),
4537                );
4538                GraftOutcome::Applied
4539            }
4540            Some(why) => {
4541                // Not settled until the restore is confirmed: the record
4542                // stays `Running`, so a resume retries it.
4543                crate::recovery::restore_repair(&wt, &pre)
4544                    .await
4545                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4546                self.state.event(
4547                    "graft",
4548                    format!("rolled back: {why}; the winner is untouched"),
4549                );
4550                GraftOutcome::RolledBack(why)
4551            }
4552        };
4553        self.state.graft = Some(GraftRecord {
4554            pre,
4555            post,
4556            lines,
4557            limit: None,
4558            agent: Some(agent_id),
4559            outcome,
4560            requested,
4561            adoption,
4562            adoption_error,
4563        });
4564        self.state.save()?;
4565        Ok(())
4566    }
4567
4568    // ----------------------------------------------------------------- fold
4569
4570    async fn fold_losers(&mut self) -> Result<()> {
4571        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4572            return Ok(());
4573        };
4574        if !self.state.candidate_preservation.is_empty() {
4575            bail!("candidate preservation incomplete; refusing cleanup");
4576        }
4577        let repo = self.state.repo.clone();
4578        let _coordination = self.coordinate(&repo).await?;
4579        let mut folded = Vec::new();
4580        for i in 0..self.state.candidates.len() {
4581            let c = &self.state.candidates[i];
4582            if c.label == winner || c.folded {
4583                continue;
4584            }
4585            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4586            remove_preserved_worktree(&repo, &wt).await?;
4587            if git::branch_exists(&repo, &branch).await?
4588                && !git::branch_delete(&repo, &branch).await?
4589            {
4590                bail!("candidate branch removal refused");
4591            }
4592            self.state.candidates[i].folded = true;
4593            self.state.save()?;
4594            folded.push(label.to_string());
4595        }
4596        // The judges are finished; their checkouts are pure cost from here.
4597        let root = self.state.worktree_root();
4598        for j in 1..=self.roles.judges.len() {
4599            let wt = root.join(format!("judge-{j}"));
4600            if wt.exists() {
4601                remove_preserved_worktree(&repo, &wt).await?;
4602            }
4603        }
4604        // The design-deliberation stage is finished by the time a tally
4605        // exists — same reasoning as the judges above.
4606        if self.state.config.graph.advise {
4607            for k in 1..=self.state.config.graph.advisors {
4608                let wt = root.join(format!("advisor-{k}"));
4609                if wt.exists() {
4610                    remove_preserved_worktree(&repo, &wt).await?;
4611                }
4612            }
4613        }
4614        if !folded.is_empty() {
4615            self.state
4616                .event("fold", format!("folded candidates {}", folded.join(", ")));
4617            self.state.save()?;
4618        }
4619        Ok(())
4620    }
4621
4622    // ------------------------------------------------------------ base sync
4623
4624    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4625    /// anything verifies it.
4626    ///
4627    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4628    /// read whatever is checked out in the winner's worktree. Left alone that
4629    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4630    /// it when the run *branched* - and a run takes long enough that the base
4631    /// has usually moved by the time it gets here. A gate that ran there
4632    /// answers "green on the commit this run started from", not "green on
4633    /// what is about to land", and the difference showed up three times in
4634    /// one day as a green run whose merge would have reverted a file another
4635    /// pull request had already landed.
4636    ///
4637    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4638    /// `land::Step::Rebase` calls, rather than a second implementation of the
4639    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4640    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4641    /// doc warns about ("two answers to a question nobody notices until a
4642    /// diff is wrong").
4643    ///
4644    /// A conflict is not the end of the road: the standing rebase is handed
4645    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4646    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4647    /// land). Once it finishes, review and the gate run as usual on the
4648    /// rebased tree, which is where a breakage the new base caused is caught
4649    /// by the ordinary gate-fix round. magi resolves nothing itself.
4650    ///
4651    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4652    /// `state.base_sync.attempts`, is how many times the base is *rebased
4653    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4654    /// *conflict* was given to a fixer. When the fixer cannot finish the
4655    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4656    /// what was tried (rounds spent, paths still conflicted) and the branch
4657    /// and worktree stay exactly as they were - untouched, for a person to
4658    /// look at - which is also what makes re-entering this function
4659    /// afterwards a no-op instead of a second attempt at the same wall. A
4660    /// push failure ends the same way.
4661    async fn sync_to_base(&mut self) -> Result<()> {
4662        if self.state.status == RunStatus::AlreadyInBase {
4663            return Ok(());
4664        }
4665        let conflicted = self
4666            .state
4667            .base_sync
4668            .as_ref()
4669            .is_some_and(|s| s.conflict.is_some());
4670        let Some(winner) = self.state.winner().cloned() else {
4671            return Ok(());
4672        };
4673
4674        let repo = self.state.repo.clone();
4675        let remote = self.state.config.merge.remote.clone();
4676        let base_branch = self.state.base_branch.clone();
4677        let tracking = format!("{remote}/{base_branch}");
4678
4679        git::fetch(&repo, &remote, &base_branch).await.ok();
4680        // No network, or the remote never had this branch: the run already
4681        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4682        // otherwise), and one that got this far is not blocked by a fetch
4683        // that fails now. It just has no newer tip to compare against.
4684        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4685            return Ok(());
4686        };
4687
4688        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4689        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4690        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4691
4692        // Before any rebase, and before a recorded conflict is honoured: a
4693        // branch whose change reached the base under other commit ids has
4694        // nothing to rebase and nothing to conflict with, and a run that
4695        // already stopped on that phantom conflict recovers here on resume.
4696        // `behind == 0` with head == tip is a branch the base has since taken
4697        // in whole, whether or not a conflict was ever recorded: the ancestry
4698        // proof must still run (`classify` ignores a head still on the start
4699        // commit).
4700        if (behind > 0 || conflicted || head == tip)
4701            && self
4702                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4703                .await?
4704        {
4705            return Ok(());
4706        }
4707        if conflicted {
4708            return Ok(());
4709        }
4710
4711        if behind == 0 {
4712            // A fixer-finished rebase moves the branch ref before the
4713            // winner's worktree is told (`sync_to_head` below). A run that
4714            // died in between resumes here with `behind == 0` and a tree still
4715            // holding the pre-rebase files, which review and the gate would
4716            // then read. That state is exactly: HEAD moved off the tip the
4717            // rebase started from, yet the tree is still identical to that
4718            // tip. A tree with edits of its own differs from it, so nothing
4719            // is thrown away.
4720            if let Some(from) = self
4721                .state
4722                .rebase_fixes
4723                .iter()
4724                .rev()
4725                .find_map(|r| r.from.clone())
4726                && from != head
4727                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4728                    .await
4729                    .is_ok_and(|o| o.ok())
4730            {
4731                git::sync_to_head(&winner.worktree).await?;
4732            }
4733            // An earlier attempt may have rebased the branch locally and died
4734            // before pushing it (only the fresh-rebase arm below pushes).
4735            // Publish it now, so the plain push at PR time is not refused as
4736            // a non-fast-forward. A run already holding a recorded conflict
4737            // never reaches here; that case is out of scope.
4738            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4739            if let Some(why) = &conflict {
4740                self.state.status = RunStatus::Blocked;
4741                self.state.event("land", why.clone());
4742            }
4743            self.state.base_sync = Some(BaseSync {
4744                tip,
4745                behind: 0,
4746                attempts,
4747                conflict,
4748                already_in: None,
4749            });
4750            self.state.save()?;
4751            return Ok(());
4752        }
4753
4754        if attempts >= BASE_SYNC_ROUNDS {
4755            let why = format!(
4756                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4757                 rebase(s); rebasing again would only race it",
4758                winner.branch
4759            );
4760            self.state.status = RunStatus::Blocked;
4761            self.state.base_sync = Some(BaseSync {
4762                tip,
4763                behind,
4764                attempts,
4765                conflict: Some(why.clone()),
4766                already_in: None,
4767            });
4768            self.state.event("land", why);
4769            self.state.save()?;
4770            return Ok(());
4771        }
4772
4773        self.state.event(
4774            "land",
4775            format!(
4776                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4777                winner.branch
4778            ),
4779        );
4780        self.state.save()?;
4781
4782        // The remote's copy of the branch, read now and only if the fetch
4783        // really succeeded (a stale tracking ref must never pin a lease). It is
4784        // pushed over after a rebase only when it is a commit this branch
4785        // already contains, by ancestry or by patch (an earlier rebase of ours
4786        // that never reached the remote): anything else is somebody else's work.
4787        let branch_tracking = format!("{remote}/{}", winner.branch);
4788        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4789        let remote_tip = if self.state.task_pr_branch().is_none()
4790            && matches!(&fetched_branch, Ok(o) if o.ok())
4791        {
4792            git::rev_parse(&repo, &branch_tracking).await.ok()
4793        } else {
4794            None
4795        };
4796        // A remote tip this branch does not contain is somebody else's work:
4797        // rebasing would leave a local tip that can never be pushed. Stop
4798        // before touching anything and say so.
4799        if let Some(theirs) = &remote_tip
4800            && !git::is_ancestor(&repo, theirs, &head).await
4801            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4802                .await
4803                .is_ok_and(|missing| missing.is_empty())
4804        {
4805            let why = format!(
4806                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4807                 them",
4808                short(theirs),
4809                winner.branch
4810            );
4811            self.state.status = RunStatus::Blocked;
4812            self.state.base_sync = Some(BaseSync {
4813                tip,
4814                behind,
4815                attempts,
4816                conflict: Some(why.clone()),
4817                already_in: None,
4818            });
4819            self.state.event("land", why);
4820            self.state.save()?;
4821            return Ok(());
4822        }
4823
4824        let scratch = self.state.dir().join("base-sync");
4825        let rebased = match crate::rebase::rebase_with_fixer(
4826            &mut self.state,
4827            &scratch,
4828            &winner.branch,
4829            &tracking,
4830        )
4831        .await
4832        {
4833            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4834            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4835            Err(e) => Err(e),
4836        };
4837        let attempts = attempts + 1;
4838        match rebased {
4839            Ok(None) => {
4840                // The branch ref moved, but a worktree that already had it
4841                // checked out (the winner's) was not told; sync its index and
4842                // files before anything reads them.
4843                git::sync_to_head(&winner.worktree).await?;
4844                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4845                let mut conflict = None;
4846                if let Some(pinned) = &remote_tip {
4847                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4848                    match pushed {
4849                        Ok(o) if o.ok() => self.state.event(
4850                            "land",
4851                            format!("pushed rebased {} to {remote}", winner.branch),
4852                        ),
4853                        Ok(o) => {
4854                            conflict = Some(format!(
4855                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4856                                winner.branch,
4857                                short(pinned),
4858                                o.stderr.chars().take(600).collect::<String>()
4859                            ));
4860                        }
4861                        Err(e) => {
4862                            conflict = Some(format!(
4863                                "rebased {} locally but could not push it: {e:#}",
4864                                winner.branch
4865                            ));
4866                        }
4867                    }
4868                }
4869                if let Some(why) = &conflict {
4870                    self.state.status = RunStatus::Blocked;
4871                    self.state.event("land", why.clone());
4872                }
4873                self.state.base_sync = Some(BaseSync {
4874                    tip: tip.clone(),
4875                    behind: 0,
4876                    attempts,
4877                    conflict,
4878                    already_in: None,
4879                });
4880                self.state
4881                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4882            }
4883            Ok(Some(conflict)) => {
4884                let why = format!(
4885                    "{} conflicts with {tracking} and did not rebase: {}",
4886                    winner.branch,
4887                    conflict.chars().take(600).collect::<String>()
4888                );
4889                self.state.status = RunStatus::Blocked;
4890                self.state.base_sync = Some(BaseSync {
4891                    tip,
4892                    behind,
4893                    attempts,
4894                    conflict: Some(why.clone()),
4895                    already_in: None,
4896                });
4897                self.state.event("land", why);
4898            }
4899            // A refused Git write or a busy repository: nothing is wrong with
4900            // the branch. The scratch rebase stays standing and the run
4901            // resumes into it (`base_sync` must not claim a conflict).
4902            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4903                self.state.save()?;
4904                return Ok(());
4905            }
4906            Err(e) => {
4907                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4908                self.state.status = RunStatus::Blocked;
4909                self.state.base_sync = Some(BaseSync {
4910                    tip,
4911                    behind,
4912                    attempts,
4913                    conflict: Some(why.clone()),
4914                    already_in: None,
4915                });
4916                self.state.event("land", why);
4917            }
4918        }
4919        self.state.save()?;
4920        Ok(())
4921    }
4922
4923    /// Push a branch an earlier attempt rebased locally but never published,
4924    /// pinned to the remote tip read right after a successful fetch. Returns
4925    /// the reason when the run must stop; `None` when there was nothing to do
4926    /// (no remote copy, the same tip, or a remote copy this branch already
4927    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4928    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4929        // Task competitions publish only after the PR text gate, to their
4930        // public head; never resurrect a candidate branch on the remote.
4931        if self.state.task_pr_branch().is_some() {
4932            return None;
4933        }
4934        let repo = self.state.repo.clone();
4935        let remote = self.state.config.merge.remote.clone();
4936        let fetched = git::fetch(&repo, &remote, branch).await;
4937        if !matches!(&fetched, Ok(o) if o.ok()) {
4938            return None;
4939        }
4940        let branch_tracking = format!("{remote}/{branch}");
4941        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4942        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4943            return None;
4944        }
4945        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4946            .await
4947            .is_ok_and(|missing| missing.is_empty())
4948        {
4949            return Some(format!(
4950                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4951                 them",
4952                short(&theirs)
4953            ));
4954        }
4955        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4956            Ok(o) if o.ok() => {
4957                self.state
4958                    .event("land", format!("pushed rebased {branch} to {remote}"));
4959                None
4960            }
4961            Ok(o) => Some(format!(
4962                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4963                 someone may have pushed): {}",
4964                short(&theirs),
4965                o.stderr.chars().take(600).collect::<String>()
4966            )),
4967            Err(e) => Some(format!(
4968                "{branch} is rebased locally but could not be pushed: {e:#}"
4969            )),
4970        }
4971    }
4972
4973    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4974    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4975    /// whether it did.
4976    ///
4977    /// Checked only when the base is ahead of the branch. A failing check is
4978    /// "not proven" - the ordinary rebase path then decides - never a reason to
4979    /// stop the run.
4980    ///
4981    /// The remote copy of the branch is held to the same standard as the local
4982    /// one: if it carries a tip this worktree does not, that tip must itself be
4983    /// proven in the base, or nothing is settled (a pull request would
4984    /// otherwise be closed over commits nobody checked). The pull request is
4985    /// closed *before* the terminal status is saved; if that fails for a
4986    /// reason other than a refusal (no network, a `gh` error) the run is left
4987    /// `Blocked` with the reason as its conflict, which a resume retries -
4988    /// the same recovery a phantom conflict gets.
4989    async fn settle_already_in(
4990        &mut self,
4991        branch: &str,
4992        tip: &str,
4993        head: &str,
4994        attempts: usize,
4995        behind: usize,
4996    ) -> Result<bool> {
4997        let repo = self.state.repo.clone();
4998        let remote = self.state.config.merge.remote.clone();
4999        let start = self.state.base_commit.clone();
5000        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5001            Ok(Some(e)) => e,
5002            Ok(None) => return Ok(false),
5003            Err(e) => {
5004                tracing::warn!("already-in-base check for {branch}: {e:#}");
5005                return Ok(false);
5006            }
5007        };
5008        let mut verified = vec![head.to_owned()];
5009        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5010        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5011        if matches!(&fetched, Ok(o) if o.ok())
5012            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5013            && theirs != head
5014        {
5015            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5016                Ok(Some(_)) => verified.push(theirs),
5017                _ => return Ok(false),
5018            }
5019        }
5020        let base_branch = self.state.base_branch.clone();
5021        let message = format!(
5022            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5023             land",
5024            evidence.names(),
5025            evidence.proof.as_str()
5026        );
5027        let closed =
5028            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5029                .await;
5030        match closed {
5031            Ok(Ok(url)) => self
5032                .state
5033                .event("land", format!("closed {url}: superseded on {base_branch}")),
5034            Ok(Err(why)) => self
5035                .state
5036                .event("land", format!("did not close a pull request: {why}")),
5037            Err(e) => {
5038                let why = format!(
5039                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5040                     not be closed ({e:#}); resume to retry"
5041                );
5042                self.state.status = RunStatus::Blocked;
5043                self.state.base_sync = Some(BaseSync {
5044                    tip: tip.to_owned(),
5045                    behind,
5046                    attempts,
5047                    conflict: Some(why.clone()),
5048                    already_in: None,
5049                });
5050                self.state.event("land", why);
5051                self.state.save()?;
5052                return Ok(true);
5053            }
5054        }
5055        self.state.status = RunStatus::AlreadyInBase;
5056        self.state.base_sync = Some(BaseSync {
5057            tip: tip.to_owned(),
5058            behind,
5059            attempts,
5060            conflict: None,
5061            already_in: Some(evidence),
5062        });
5063        self.state.event("land", message);
5064        self.state.save()?;
5065        self.settle_questions();
5066        Ok(true)
5067    }
5068
5069    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5070    /// last landed the winner on, once it has run, else the commit the run
5071    /// branched from.
5072    ///
5073    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5074    /// and `vote` all happen before there is a winner to rebase, so they
5075    /// compare every candidate against the branch point on purpose, and a
5076    /// base that moves after they are already done cannot change an answer
5077    /// they already gave.
5078    fn landing_base(&self) -> String {
5079        self.state
5080            .base_sync
5081            .as_ref()
5082            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5083    }
5084
5085    // ------------------------------------------------------- operator fix
5086
5087    /// Route specific, already-recorded review findings to a fixer for a
5088    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5089    /// entry point.
5090    ///
5091    /// Distinct from `review_loop`'s own fix step in three ways: it never
5092    /// runs a reviewer wave, it never spends review-round budget, and what
5093    /// happened is recorded as an [`OperatorFixRequest`] appended to
5094    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5095    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5096    /// and vote must never be rewritten to look like a manufactured blocking
5097    /// verdict.
5098    ///
5099    /// Only meaningful once review has actually concluded: `Ready` (handed
5100    /// off with findings still open, or simply concluded clean while minor
5101    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5102    /// gate failed). Everything else is refused: a run still in progress
5103    /// should simply be resumed, and a `Merged` run's branch has already
5104    /// landed — reopening *this* run's own record cannot change that, so the
5105    /// answer there is a fresh `magi review <branch>`.
5106    ///
5107    /// A real commit here re-verifies through a fresh, ordinary review-only
5108    /// run on the same branch ([`Self::review`]) rather than reopening this
5109    /// run's own `review_loop`: once any round in this run's history went
5110    /// clean, `review_conclusion` treats that as permanent by design (the
5111    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5112    /// way to force one more genuine reviewer wave out of *this* run without
5113    /// either rewriting history or weakening that guarantee for every other
5114    /// caller. A review-only run costs nothing extra — no implementation, no
5115    /// judging, no vote — and exercises the exact same review → verify →
5116    /// gate → (human) merge path, unmodified.
5117    pub async fn fix_selected(
5118        &mut self,
5119        ids: &[String],
5120        reason: &str,
5121        allow_stale: bool,
5122    ) -> Result<()> {
5123        let reason = reason.trim();
5124        if reason.is_empty() {
5125            bail!("a fix request needs a reason — that is the operator's own record of why");
5126        }
5127        if ids.is_empty() {
5128            bail!("no finding id given");
5129        }
5130        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5131            bail!(
5132                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5133                 has already concluded — can be given a targeted fix. A run still \
5134                 in progress should simply be resumed; a `merged` run's branch has \
5135                 already landed, so its answer is a fresh `magi review <branch>`, \
5136                 not reopening this run's own record",
5137                self.state.id,
5138                self.state.status.as_str()
5139            );
5140        }
5141        let Some(winner) = self.state.winner().cloned() else {
5142            bail!("run {} has no winning candidate to fix", self.state.id);
5143        };
5144        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5145            bail!(
5146                "branch `{}` no longer exists; this run cannot be extended",
5147                winner.branch
5148            );
5149        }
5150        let home = crate::run::home();
5151        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5152            bail!(
5153                "run {} is currently being worked on by another magi process",
5154                self.state.id
5155            );
5156        }
5157        // Held for the rest of this call, including the follow-up review
5158        // below: two `magi fix` invocations against the same run must not
5159        // both reach the worktree manipulation further down, which would
5160        // otherwise race to remove and recreate the same directory — see
5161        // [`FixClaim`]'s own doc.
5162        let _claim = FixClaim::acquire(&self.state.dir())?;
5163
5164        // Resolve every id before spending anything — an unknown id refuses
5165        // the whole request rather than silently dropping it — and dedup
5166        // while keeping the operator's own order.
5167        let mut seen = BTreeSet::new();
5168        let mut findings = Vec::new();
5169        let mut missing = Vec::new();
5170        for id in ids {
5171            if !seen.insert(id.clone()) {
5172                continue;
5173            }
5174            match self.state.finding(id) {
5175                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5176                    id: f.id.clone(),
5177                    severity: f.severity,
5178                    reviewer_vote: rec.vote,
5179                    round: round.round,
5180                    round_head: round.head.clone(),
5181                    reviewer: rec.reviewer,
5182                    agent: rec.agent.clone(),
5183                    file: f.file.clone(),
5184                    line: f.line,
5185                    title: f.title.clone(),
5186                    detail: f.detail.clone(),
5187                    outcome: OperatorFixOutcome::Pending,
5188                }),
5189                None => missing.push(id.clone()),
5190            }
5191        }
5192        if !missing.is_empty() {
5193            bail!(
5194                "unknown finding id(s): {}; nothing was changed",
5195                missing.join(", ")
5196            );
5197        }
5198
5199        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5200        let stale_details: Vec<(String, String)> = findings
5201            .iter()
5202            .filter(|f| f.round_head != head_at_request)
5203            .map(|f| (f.id.clone(), f.round_head.clone()))
5204            .collect();
5205        let stale = !stale_details.is_empty();
5206        if stale && !allow_stale {
5207            bail!(
5208                "the branch has moved since some finding(s) were raised — {} — now \
5209                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5210                stale_details
5211                    .iter()
5212                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5213                    .collect::<Vec<_>>()
5214                    .join(", "),
5215                short(&head_at_request)
5216            );
5217        }
5218
5219        let request = OperatorFixRequest {
5220            requested_at: Timestamp::now(),
5221            reason: reason.to_owned(),
5222            findings,
5223            head_at_request: head_at_request.clone(),
5224            allow_stale,
5225            stale,
5226            fix: None,
5227            result_head: None,
5228            follow_up_review_run: None,
5229        };
5230        self.state.event(
5231            "fix",
5232            format!(
5233                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5234                request.findings.len(),
5235                request
5236                    .findings
5237                    .iter()
5238                    .map(|f| f.id.as_str())
5239                    .collect::<Vec<_>>()
5240                    .join(", "),
5241            ),
5242        );
5243        // Recorded now, before any worktree work or the fixer call itself —
5244        // and re-saved at each checkpoint below: a crash at any point after
5245        // this (mid fixer call, mid follow-up review) must not lose the fact
5246        // that this was requested, for which findings, and why. Everything
5247        // past this point reads and writes through `request_index` rather
5248        // than a local variable, since `request` itself is moved here.
5249        self.state.operator_fixes.push(request);
5250        self.state.save()?;
5251        let request_index = self.state.operator_fixes.len() - 1;
5252
5253        // A fresh, dedicated worktree for this one call, never the winner's
5254        // own worktree in place: that one may already be gone (folded away),
5255        // and reusing it in place would leave the branch checked out there
5256        // when the follow-up review below tries to check it out again. Freed
5257        // immediately after, either way — but only once confirmed clean:
5258        // `worktree_remove` is a `git worktree remove --force`, which would
5259        // otherwise discard uncommitted work left there by the operator or
5260        // another process before this had a chance to even look at it.
5261        let repo = self.state.repo.clone();
5262        let coordination = self.coordinate(&repo).await?;
5263        if winner.worktree.exists() {
5264            // Lockfiles a rescue commit withheld stay untracked on purpose and
5265            // are already recorded; they are not the operator's work to protect.
5266            let dirty = git::git(
5267                &winner.worktree,
5268                &["status", "--porcelain", "--untracked-files=all"],
5269            )
5270            .await?;
5271            let only_withheld = dirty.lines().all(|l| {
5272                l.strip_prefix("?? ")
5273                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5274            });
5275            if !only_withheld {
5276                bail!(
5277                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5278                     discard them first",
5279                    winner.worktree.display()
5280                );
5281            }
5282            git::worktree_remove(&self.state.repo, &winner.worktree)
5283                .await
5284                .ok();
5285        }
5286        let fix_worktree = self.state.worktree_root().join("operator-fix");
5287        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5288        git::git(
5289            &self.state.repo,
5290            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5291        )
5292        .await
5293        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5294        if !git::is_clean(&fix_worktree).await? {
5295            git::worktree_remove(&self.state.repo, &fix_worktree)
5296                .await
5297                .ok();
5298            bail!(
5299                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5300                winner.branch
5301            );
5302        }
5303        let repo_for_setup = self.state.repo.clone();
5304        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5305            Ok(()) => {
5306                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5307            }
5308            Err(e) => Err(e),
5309        };
5310        if let Err(e) = setup {
5311            git::worktree_remove(&self.state.repo, &fix_worktree)
5312                .await
5313                .ok();
5314            return Err(e);
5315        }
5316
5317        let run_id = self.state.id.clone();
5318        let prompts = self.state.config.prompts.clone();
5319        let language = self.state.config.graph.language.clone();
5320        let sessions = self.state.config.graph.sessions;
5321        let artifacts = agent::artifacts_dir(&self.state.dir());
5322        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5323            .findings
5324            .iter()
5325            .map(|f| Finding {
5326                id: f.id.clone(),
5327                severity: f.severity,
5328                file: f.file.clone(),
5329                line: f.line,
5330                title: f.title.clone(),
5331                detail: f.detail.clone(),
5332            })
5333            .collect();
5334        let fix_prompt = prompt::operator_fix_in(
5335            &self.state.instruction,
5336            &finding_list,
5337            reason,
5338            &stale_details,
5339            &head_at_request,
5340            &language,
5341            &self.state.config.graph.github_language,
5342        );
5343        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5344        drop(coordination);
5345        let (job, seat, out) = self
5346            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5347                prompt: fix_prompt.clone(),
5348                spec,
5349                seat,
5350                cwd: fix_worktree.clone(),
5351                timeout,
5352                allow_write: true,
5353                sessions,
5354                artifacts: artifacts.clone(),
5355                stem: "operator-fix".to_owned(),
5356                handover: None,
5357            })
5358            .await;
5359        let agent_id = seat.agent.clone();
5360
5361        let mut fix = FixRecord {
5362            agent: agent_id,
5363            addressed: Vec::new(),
5364            rejected: Vec::new(),
5365            notes: String::new(),
5366            committed: false,
5367            failed: None,
5368            duration_ms: 0,
5369            continuation: None,
5370        };
5371        let mut final_seat = seat.clone();
5372        match out {
5373            AgentOutcome::Ok(o) => {
5374                fix.duration_ms = o.duration_ms;
5375                let parsed = verdict::extract_json::<FixReport>(&o.text);
5376                let incomplete_reason = match &parsed {
5377                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5378                        "the reply parsed, but it reported a command whose own CLI \
5379                         never confirmed an exit status"
5380                            .to_owned(),
5381                    ),
5382                    Ok(_) => None,
5383                    Err(e) => Some(e.to_string()),
5384                };
5385                match incomplete_reason {
5386                    None => {
5387                        let report = parsed.expect("checked Ok above");
5388                        fix.addressed = report.addressed;
5389                        fix.rejected = report.rejected;
5390                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5391                    }
5392                    Some(reason) => {
5393                        let (resumed_seat, resolved, failure, cont) = self
5394                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5395                            .await;
5396                        fix.duration_ms += cont.cumulative_wait_ms;
5397                        fix.continuation = Some(cont);
5398                        final_seat = resumed_seat;
5399                        match resolved {
5400                            Some(report) => {
5401                                fix.addressed = report.addressed;
5402                                fix.rejected = report.rejected;
5403                                fix.notes =
5404                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5405                            }
5406                            None => fix.failed = failure,
5407                        }
5408                    }
5409                }
5410            }
5411            AgentOutcome::Dropped(o) => {
5412                fix.duration_ms = o.duration_ms;
5413                let why = o
5414                    .dropped
5415                    .as_ref()
5416                    .map(|d| d.why.as_str())
5417                    .unwrap_or("the CLI ended the stream without delivering its answer");
5418                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5419            }
5420            AgentOutcome::Quota(o) => {
5421                self.state.quota.push(QuotaLoss {
5422                    seat: final_seat.key.clone(),
5423                    node: "fix".to_owned(),
5424                    at: Timestamp::now(),
5425                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5426                });
5427                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5428            }
5429            AgentOutcome::Failed(e) => fix.failed = Some(e),
5430            AgentOutcome::ResourceBlocked(e) => {
5431                fix.failed = Some(format!("the fixer was not started: {e}"));
5432            }
5433        }
5434        if fix.continuation.is_none() {
5435            fix.continuation = Some(ContinuationRecord::not_needed());
5436        }
5437        self.state.seats.insert(final_seat.key.clone(), final_seat);
5438
5439        let rescue_message = format!(
5440            "magi: operator-selected fix ({}) (uncommitted work)",
5441            self.state.operator_fixes[request_index]
5442                .findings
5443                .iter()
5444                .map(|f| f.id.as_str())
5445                .collect::<Vec<_>>()
5446                .join(", ")
5447        );
5448        let repo = self.state.repo.clone();
5449        let coordination = self.coordinate(&repo).await?;
5450        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5451            self.state.note_withheld("fix", &r.withheld);
5452        }
5453        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5454        fix.committed = after != head_at_request;
5455        git::worktree_remove(&self.state.repo, &fix_worktree)
5456            .await
5457            .ok();
5458        drop(coordination);
5459
5460        self.state.event(
5461            "fix",
5462            match &fix.failed {
5463                Some(reason) => format!(
5464                    "operator fix: adoption report was lost ({reason}); {}",
5465                    if fix.committed {
5466                        "committed"
5467                    } else {
5468                        "NO new commit"
5469                    }
5470                ),
5471                None => format!(
5472                    "operator fix: {} addressed, {} rejected, {}",
5473                    fix.addressed.len(),
5474                    fix.rejected.len(),
5475                    if fix.committed {
5476                        "committed"
5477                    } else {
5478                        "NO new commit"
5479                    }
5480                ),
5481            },
5482        );
5483
5484        // Every selected finding gets an outcome — never left `Pending` once
5485        // the fixer's own turn is over. A report that never came back at all
5486        // marks every one of them `Unreported`, not silently "not addressed":
5487        // quota, a dropped stream, or an exhausted continuation are gaps in
5488        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5489        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5490        for f in &mut self.state.operator_fixes[request_index].findings {
5491            f.outcome = if fix.failed.is_some() {
5492                OperatorFixOutcome::Unreported
5493            } else if fix.addressed.contains(&f.id) {
5494                OperatorFixOutcome::Addressed
5495            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5496                OperatorFixOutcome::Rejected { why: r.why.clone() }
5497            } else {
5498                OperatorFixOutcome::Unreported
5499            };
5500        }
5501
5502        let committed = fix.committed;
5503        if committed {
5504            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5505        }
5506        self.state.operator_fixes[request_index].fix = Some(fix);
5507        // Saved again now that the fixer's own outcome is final, on top of
5508        // the save right after the request was first pushed above.
5509        self.state.save()?;
5510
5511        if committed {
5512            self.state.event(
5513                "fix",
5514                format!(
5515                    "operator fix committed {}; opening a follow-up review-only run",
5516                    short(&after)
5517                ),
5518            );
5519            // The operator asked for the fix, and the follow-up serves whatever
5520            // task the run it follows served.
5521            let origin =
5522                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5523            match Self::review(
5524                &self.state.repo,
5525                &winner.branch,
5526                self.state.config.clone(),
5527                origin,
5528            )
5529            .await
5530            {
5531                Ok(mut follow_up) => {
5532                    follow_up.state.event(
5533                        "start",
5534                        format!(
5535                            "requested by an operator fix on run {} for finding(s) {}",
5536                            self.state.id,
5537                            self.state.operator_fixes[request_index]
5538                                .findings
5539                                .iter()
5540                                .map(|f| f.id.as_str())
5541                                .collect::<Vec<_>>()
5542                                .join(", "),
5543                        ),
5544                    );
5545                    follow_up.state.save()?;
5546                    let follow_up_id = follow_up.state.id.clone();
5547                    // The follow-up is a run like any other: it belongs to the
5548                    // task of the run it follows, or to one filed for it.
5549                    let adopted = match crate::direct::adopt(
5550                        &follow_up.state,
5551                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5552                    ) {
5553                        Ok(a) => a,
5554                        Err(e) => {
5555                            // An ownerless run must not spend agent calls; it
5556                            // stays saved, and `magi run --resume` adopts it.
5557                            self.state.event(
5558                                "fix",
5559                                format!(
5560                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5561                                ),
5562                            );
5563                            self.state.save()?;
5564                            return Ok(());
5565                        }
5566                    };
5567                    let executed = follow_up.execute().await;
5568                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5569                    if let Some(a) = adopted {
5570                        a.finish(&follow_up.state, executed);
5571                    }
5572                    if let Some(e) = failure {
5573                        self.state.event(
5574                            "fix",
5575                            format!(
5576                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5577                            ),
5578                        );
5579                    }
5580                    self.state.operator_fixes[request_index].follow_up_review_run =
5581                        Some(follow_up_id);
5582                }
5583                Err(e) => {
5584                    self.state.event(
5585                        "fix",
5586                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5587                    );
5588                }
5589            }
5590            self.state.save()?;
5591        }
5592
5593        Ok(())
5594    }
5595
5596    // --------------------------------------------------------------- review
5597
5598    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5599    /// names, in order, each at most once, moving on only when the call
5600    /// advances (an error, a quota hit or nothing usable - the same decision
5601    /// point `agent::chain_advances` is for the other chained roles).
5602    ///
5603    /// Each agent gets its own seat from `Runner::seat`: the same agent
5604    /// continues its conversation (the winner's own implementer seat when it
5605    /// is the winner's author, now that the competition is over), another
5606    /// takes a fresh one so the full prompt is sent again. A handover is
5607    /// recorded under `node`, which is what makes the fallback stick for the
5608    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5609    /// is returned, so an exhausted chain reads exactly like a single failed
5610    /// fixer: one quota loss, the same wording, the same refund. An earlier
5611    /// attempt's edits are left in the tree and judged, with the final
5612    /// attempt's, by what git says afterwards.
5613    ///
5614    /// Returns the job that produced the outcome, for `continue_fix_report`.
5615    async fn ask_fixer(
5616        &mut self,
5617        winner: &Candidate,
5618        node: &'static str,
5619        round: Option<usize>,
5620        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5621    ) -> (SeatJob, SeatState, AgentOutcome) {
5622        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5623        self.ask_fixer_among(attempts, node, round, None, build)
5624            .await
5625    }
5626
5627    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5628    /// worktree and a commit: before the next agent is asked after one that
5629    /// did not satisfy the call, the worktree is put back on that commit, so
5630    /// a failed agent's partial edits are not carried into the next one.
5631    async fn ask_fixer_among(
5632        &mut self,
5633        attempts: Vec<(AgentSpec, String)>,
5634        node: &'static str,
5635        round: Option<usize>,
5636        restore: Option<(&Path, &str)>,
5637        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5638    ) -> (SeatJob, SeatState, AgentOutcome) {
5639        let run_id = self.state.id.clone();
5640        let prompts = self.state.config.prompts.clone();
5641        let cache = self.state.config.cache_dir();
5642        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5643        let mut last = None;
5644        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5645            let seat = self.seat(&key, &spec.id);
5646            let mut job = build(spec.clone(), seat);
5647            if i > 0 {
5648                job.stem = format!("{}-{}", job.stem, spec.id);
5649            }
5650            let ctx = WaveCtx {
5651                carry_seats: false,
5652                pauses: Some((&self.pause, &self.interrupt)),
5653                run: &run_id,
5654                node,
5655                prompts: &prompts,
5656                cache: cache.as_deref(),
5657                round,
5658            };
5659            let (seat, out) =
5660                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5661            // A resource block never advances the chain: the chosen agent
5662            // was not asked, so it must be the one asked again on resume.
5663            let advances = match &out {
5664                AgentOutcome::Ok(o) => agent::output_advances(o),
5665                AgentOutcome::ResourceBlocked(_) => false,
5666                _ => true,
5667            };
5668            if let Some(next) = ids.get(i + 1)
5669                && advances
5670            {
5671                self.state.seats.insert(seat.key.clone(), seat.clone());
5672                let class =
5673                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5674                let reason = match &out {
5675                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5676                    other => fail_reason(other),
5677                };
5678                // A failed agent's edits must not reach the next one, and a
5679                // restore that did not work ends the chain: the call fails
5680                // and the caller's own (confirmed) restore decides.
5681                if let Some((wt, rev)) = restore
5682                    && let Err(e) = self.graft_restore(wt, rev).await
5683                {
5684                    let why =
5685                        format!("could not restore the worktree before the next fixer: {e:#}");
5686                    tracing::warn!("{why}");
5687                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5688                        AgentOutcome::ResourceBlocked(why)
5689                    } else {
5690                        AgentOutcome::Failed(why)
5691                    };
5692                    last = Some((job, seat, out));
5693                    break;
5694                }
5695                record_handover(
5696                    &mut self.state,
5697                    node,
5698                    &seat.key,
5699                    &spec.id,
5700                    next,
5701                    &class,
5702                    &reason,
5703                );
5704                continue;
5705            }
5706            last = Some((job, seat, out));
5707            break;
5708        }
5709        last.expect("the fixer chain always has an entry")
5710    }
5711
5712    /// The one repair a verification-only failure earns (see
5713    /// [`crate::recovery`]), before the first review round of a run that took
5714    /// a branch over.
5715    ///
5716    /// Saved as started *before* the fixer is called, so a resume never
5717    /// repairs twice; a start found on re-entry is rolled back to the commit
5718    /// it began from. A failed or unusable fixer leaves the tree as it was and
5719    /// the review goes on over it, so the first round's own verification
5720    /// reports the failure again. The failure is never carried into a round as
5721    /// a result: verification is re-run on whatever commit this leaves.
5722    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5723        use crate::recovery::{Mode, Repair};
5724        if !self.state.reviews.is_empty() {
5725            return Ok(());
5726        }
5727        let Some(mut rec) = self.state.recovery.clone() else {
5728            return Ok(());
5729        };
5730        if rec.mode != Mode::VerifyRepair {
5731            return Ok(());
5732        }
5733        let wt = winner.worktree.clone();
5734        let mut head = git::rev_parse(&wt, "HEAD").await?;
5735        match rec.repair.clone() {
5736            Repair::Done { .. } => return Ok(()),
5737            Repair::WaitingForRestore { before } => {
5738                self.graft_restore(&wt, &before).await?;
5739                head = before;
5740                rec.repair = Repair::NotStarted;
5741                self.state.recovery = Some(rec.clone());
5742                self.state.save()?;
5743            }
5744            Repair::Started { before } => {
5745                // Interrupted: the unfinished edits are not trusted.
5746                // An unconfirmed rollback blocks the run with `Started` kept,
5747                // so a resume retries it; the tree is never reviewed as is.
5748                if let Err(e) = self.graft_restore(&wt, &before).await {
5749                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5750                        return Err(e);
5751                    }
5752                    return self
5753                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5754                        .await;
5755                }
5756                rec.repair = Repair::Done {
5757                    before: before.clone(),
5758                    after: before,
5759                    committed: false,
5760                    failed: Some("interrupted; rolled back".to_owned()),
5761                };
5762                self.state
5763                    .event("recovery", "the repair was interrupted and rolled back");
5764                self.state.recovery = Some(rec);
5765                self.state.save()?;
5766                return Ok(());
5767            }
5768            Repair::NotStarted => {}
5769        }
5770        if crate::recovery::recheck(&mut rec, &head) {
5771            self.state.event("recovery", rec.summary());
5772            self.state.recovery = Some(rec);
5773            self.state.save()?;
5774            return Ok(());
5775        }
5776        rec.repair = Repair::Started {
5777            before: head.clone(),
5778        };
5779        self.state.recovery = Some(rec.clone());
5780        self.state.save()?;
5781
5782        let language = self.state.config.graph.language.clone();
5783        let sessions = self.state.config.graph.sessions;
5784        let artifacts = agent::artifacts_dir(&self.state.dir());
5785        let fix_prompt = prompt::recovery_fix(
5786            &self.state.instruction,
5787            &rec,
5788            self.state.config.graph.review_rounds,
5789            &language,
5790        );
5791        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5792        self.state.event(
5793            "recovery",
5794            format!(
5795                "repair: the fixer starts from the failed verification on {}",
5796                short(&head)
5797            ),
5798        );
5799        let (_job, seat, out) = self
5800            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5801                prompt: fix_prompt.clone(),
5802                spec,
5803                seat,
5804                cwd: wt.clone(),
5805                timeout,
5806                allow_write: true,
5807                sessions,
5808                artifacts: artifacts.clone(),
5809                stem: "fix-recovery".to_owned(),
5810                handover: None,
5811            })
5812            .await;
5813        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5814            rec.repair = Repair::WaitingForRestore { before: head };
5815            self.state.recovery = Some(rec);
5816            return self.stop_for_resource();
5817        }
5818        let failed = match &out {
5819            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5820            AgentOutcome::Quota(o) => {
5821                self.state.quota.push(QuotaLoss {
5822                    seat: seat.key.clone(),
5823                    node: "fix".to_owned(),
5824                    at: Timestamp::now(),
5825                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5826                });
5827                Some("rate limited (quota); the fixer could not run".to_owned())
5828            }
5829            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5830            AgentOutcome::Failed(e) => Some(e.clone()),
5831        };
5832        self.state.seats.insert(seat.key.clone(), seat);
5833        let coordination = self.coordinate(&wt).await?;
5834        let failed = match failed {
5835            // The agent's partial edits are not kept.
5836            Some(why) => {
5837                if let Err(e) = git::restore_to(&wt, &head).await {
5838                    return self
5839                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5840                        .await;
5841                }
5842                Some(why)
5843            }
5844            None => {
5845                if let Ok(r) = git::rescue_commit(
5846                    &wt,
5847                    "magi: recovery repair of the failed verification (uncommitted work)",
5848                )
5849                .await
5850                {
5851                    self.state.note_withheld("fix", &r.withheld);
5852                }
5853                None
5854            }
5855        };
5856        let after = git::rev_parse(&wt, "HEAD").await?;
5857        drop(coordination);
5858        self.state.event(
5859            "recovery",
5860            match &failed {
5861                Some(why) => format!("repair: nothing kept ({why})"),
5862                None if after != head => {
5863                    format!("repair: committed {}", short(&after))
5864                }
5865                None => "repair: the fixer made no commit".to_owned(),
5866            },
5867        );
5868        rec.repair = Repair::Done {
5869            before: head.clone(),
5870            committed: after != head,
5871            after,
5872            failed,
5873        };
5874        self.state.recovery = Some(rec);
5875        self.state.save()?;
5876        Ok(())
5877    }
5878
5879    /// A repair whose rollback could not be confirmed: the run is blocked and
5880    /// resumable, and the unconfirmed state is never reviewed.
5881    async fn stop_recovery(
5882        &mut self,
5883        mut rec: crate::recovery::RecoveryRecord,
5884        head: &str,
5885        why: String,
5886    ) -> Result<()> {
5887        // `Started` stays recorded: a resume retries the restore.
5888        rec.repair = crate::recovery::Repair::Started {
5889            before: head.to_owned(),
5890        };
5891        self.state.recovery = Some(rec);
5892        self.state
5893            .event("recovery", format!("repair could not be undone: {why}"));
5894        self.state.status = RunStatus::Blocked;
5895        self.state.save()?;
5896        bail!("recovery repair could not be rolled back: {why}")
5897    }
5898
5899    async fn review_loop(&mut self) -> Result<()> {
5900        loop {
5901            if self
5902                .state
5903                .base_sync
5904                .as_ref()
5905                .is_some_and(|s| s.conflict.is_some())
5906            {
5907                return Ok(());
5908            }
5909            if self.catchup_repairable() {
5910                let record = self.state.reviews.last().expect("repairable round").clone();
5911                let winner = self
5912                    .state
5913                    .winner()
5914                    .cloned()
5915                    .context("catch-up repair has no winner")?;
5916                if !self.fix_review_round(&winner, record).await? {
5917                    return Ok(());
5918                }
5919            }
5920            self.review_loop_pass().await?;
5921            if !self.catchup_repairable() {
5922                return Ok(());
5923            }
5924        }
5925    }
5926
5927    /// Derive an owed catch-up repair from durable round evidence, including
5928    /// reentry after a crash before the fixer was dispatched.
5929    fn catchup_repairable(&self) -> bool {
5930        self.state.reviews.len() < self.state.config.graph.review_rounds
5931            && self.state.reviews.last().is_some_and(|r| {
5932                !r.clean
5933                    && r.blocking == 0
5934                    && r.fix.is_none()
5935                    && r.e2e_status() == E2eStatus::Failed
5936            })
5937    }
5938
5939    async fn review_loop_pass(&mut self) -> Result<()> {
5940        // A base that would not rebase is a person's decision, not a review
5941        // round: nothing here would change the answer, and reviewers and a
5942        // fixer would be spending real budget on a tree that cannot land
5943        // regardless of what they find.
5944        if self
5945            .state
5946            .base_sync
5947            .as_ref()
5948            .is_some_and(|s| s.conflict.is_some())
5949        {
5950            return Ok(());
5951        }
5952        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5953        // agent files with `magi task add` name the run that paid for it. The
5954        // prompt overlay is cloned alongside it because the waves borrow it
5955        // while `self` is mutably borrowed by the node's own bookkeeping.
5956        let run_id = self.state.id.clone();
5957        let prompts = self.state.config.prompts.clone();
5958        let Some(winner) = self.state.winner().cloned() else {
5959            return Ok(());
5960        };
5961        let max_rounds = self.state.config.graph.review_rounds;
5962        // A clean round, an exhausted round budget, or a stalled tree (see
5963        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5964        // they are recorded — recomputed here, not read off `status`, so a
5965        // reentry into a run that already stopped restates the identical
5966        // verdict instead of silently handing back whatever an earlier node
5967        // in this same walk clobbered `status` to (a solo-candidate
5968        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5969        // below runs an empty range once the budget is spent, and would
5970        // otherwise fall through without touching `status` at all.
5971        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5972            // A reentry after a crash between the last round's save and
5973            // `stop_reviewing` reaches the hand-off here, not there.
5974            if status == RunStatus::Gating {
5975                self.record_contested_handoff();
5976            }
5977            self.state.status = status;
5978            self.state.save()?;
5979            return Ok(());
5980        }
5981        self.state.status = RunStatus::Reviewing;
5982        // A last recorded round whose own verification never resolved
5983        // (`ResourceBlocked` — the shared build cache, not the patch) is
5984        // never a concluded round, whatever the round budget says: starting
5985        // a fresh round on top of it would spend a whole new reviewer wave
5986        // re-reading an unchanged patch instead of just retrying the one
5987        // check that actually needs it, and once the budget is spent the
5988        // loop below has nothing left to do at all (its range is empty).
5989        // Retry that check directly instead, exactly the same retry
5990        // `stop_reviewing` already does for its own catch-up case.
5991        if self.state.reviews.last().is_some_and(|r| {
5992            r.e2e_status() == E2eStatus::ResourceBlocked
5993                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5994        }) {
5995            let shell = self.state.config.shell();
5996            return self
5997                .stop_unresolved_review(
5998                    "the last round's own verification never resolved",
5999                    &shell,
6000                    &winner.worktree,
6001                )
6002                .await;
6003        }
6004
6005        self.recovery_repair(&winner).await?;
6006        if self.resource_halt()? {
6007            return Ok(());
6008        }
6009
6010        let repo = self.state.repo.clone();
6011        let root = self.state.worktree_root();
6012        let language = self.state.config.graph.language.clone();
6013        let sessions = self.state.config.graph.sessions;
6014        let artifacts = self.merge_repair_artifacts();
6015        let base = self.landing_base();
6016        let base_short = short(&base);
6017        let reviewers = self.roles.reviewers.clone();
6018        let shell = self.state.config.shell();
6019
6020        for round in (self.state.reviews.len() + 1)..=max_rounds {
6021            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6022            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6023            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6024            // The prior round's own record, already persisted — never a
6025            // hand-carried variable of just its failing output: that is
6026            // exactly what let a round's e2e result drift out of sync with
6027            // which commit it was actually about (see `SCHEMA`'s doc for
6028            // schema 8). Judged against `head`, the commit reviewers are
6029            // about to look at now, so the summary always reads as "an
6030            // earlier head" here — this round's own patch has not been
6031            // checked yet.
6032            let prev_verification = self
6033                .state
6034                .reviews
6035                .last()
6036                .and_then(|r| r.verification_summary(&head));
6037            // Only the first round of a run that took a branch over: later
6038            // rounds have this run's own history.
6039            let prior_section = match (&self.state.recovery, round) {
6040                (Some(rec), 1) if self.state.reviews.is_empty() => {
6041                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6042                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6043                            Some(crate::recovery::head_of(
6044                                &git::diff(&winner.worktree, reviewed, "HEAD")
6045                                    .await
6046                                    .unwrap_or_default(),
6047                                crate::recovery::SINCE_DIFF_MAX,
6048                            ))
6049                        }
6050                        _ => None,
6051                    };
6052                    Some(prompt::recovery_review(rec, since.as_deref()))
6053                }
6054                _ => None,
6055            };
6056
6057            // Each reviewer gets its own detached checkout of exactly this
6058            // commit: nobody can perturb the winner's tree, and the fixer can
6059            // keep working without racing a reviewer.
6060            let mut jobs = Vec::new();
6061            let coordination = self.coordinate(&repo).await?;
6062            for (r, spec) in reviewers.iter().cloned().enumerate() {
6063                let wt = root.join(format!("review-{}", r + 1));
6064                // `clean -fdx` in a reset wipes setup's products, so every
6065                // round sets the seat up again.
6066                if wt.exists() {
6067                    git::reset_detached(&wt, &head).await?;
6068                } else {
6069                    git::worktree_add_detached(&repo, &wt, &head).await?;
6070                }
6071                self.setup_seat_worktree(&repo, &wt, false).await?;
6072                let seat_key = format!("review-{}", r + 1);
6073                // The seat starts the round on whoever answered it last, not
6074                // on the agent the spec names, so a failure is not re-paid.
6075                let spec = pick_start_spec(
6076                    &self.roles.reviewer_roster,
6077                    spec,
6078                    self.state.seat_history.get(&seat_key),
6079                );
6080                let seat = self.seat(&seat_key, &spec.id);
6081                jobs.push(SeatJob {
6082                    prompt: prompt::review(&prompt::ReviewCtx {
6083                        instruction: &self.state.instruction,
6084                        branch: &winner.branch,
6085                        base_short: &base_short,
6086                        stat: &stat,
6087                        patch: &patch,
6088                        verification: prev_verification.as_ref(),
6089                        prior: prior_section.as_deref(),
6090                        reviewers: reviewers.len(),
6091                        round,
6092                        rounds: max_rounds,
6093                        // A review-only run has no rankings, so nothing
6094                        // competed for this patch and the reviewer is told so.
6095                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6096                        lens: Lens::for_seat(r),
6097                        language: &language,
6098                        github_language: &self.state.config.graph.github_language,
6099                    }),
6100                    spec,
6101                    seat,
6102                    cwd: wt,
6103                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6104                    allow_write: false,
6105                    sessions,
6106                    artifacts: artifacts.clone(),
6107                    stem: format!("review-{round}-{}", r + 1),
6108                    handover: None,
6109                });
6110            }
6111
6112            drop(coordination);
6113            self.state.event(
6114                "review",
6115                format!(
6116                    "round {round}: {} reviewers on {}",
6117                    jobs.len(),
6118                    short(&head)
6119                ),
6120            );
6121            let mut quota_losses = Vec::new();
6122            let review_retries = self.state.config.graph.retries;
6123            let review_cache = self.state.config.cache_dir();
6124            let ctx = WaveCtx {
6125                carry_seats: true,
6126                pauses: Some((&self.pause, &self.interrupt)),
6127                run: &run_id,
6128                node: "review",
6129                prompts: &prompts,
6130                cache: review_cache.as_deref(),
6131                round: Some(round),
6132            };
6133            let results = ask_json_wave::<Review>(
6134                jobs,
6135                Arc::clone(&self.sem),
6136                review_retries,
6137                &self.roles.reviewer_roster,
6138                &ctx,
6139                &mut quota_losses,
6140                &mut self.state,
6141                &|_: &Review| Ok(()),
6142            )
6143            .await;
6144            if self.state.resource_block.is_some() {
6145                return self.stop_for_resource();
6146            }
6147            // Counted before the move below: how many of *this* round's
6148            // reviewer seats were lost to their own rate limit, as opposed to
6149            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6150            let round_quota_missing = quota_losses.len();
6151            self.state.quota.extend(quota_losses);
6152
6153            let mut records = Vec::new();
6154            let mut all_findings = Vec::new();
6155            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6156                let agent_id = seat.agent.clone();
6157                self.state.seats.insert(seat.key.clone(), seat);
6158                let mut record = ReviewRecord {
6159                    reviewer: r + 1,
6160                    agent: agent_id,
6161                    summary: String::new(),
6162                    findings: Vec::new(),
6163                    vote: None,
6164                    failed: None,
6165                    duration_ms: 0,
6166                    // Set for both outcomes: `failed: Some(_)` with
6167                    // `attempts > 0` is a seat every retry still lost, not a
6168                    // recovered one — only `failed: None` with `attempts > 0`
6169                    // reads as "answered after a nudge" (see this field's own
6170                    // doc).
6171                    attempts,
6172                };
6173                match res {
6174                    Ok((review, out)) => {
6175                        // Sanitized here, at the point every other piece of
6176                        // agent prose in this file is (candidate summaries,
6177                        // deliberation turns, vote reasons): a reviewer's own
6178                        // words are the one thing about it that could name
6179                        // it, and reconsideration below broadcasts this same
6180                        // summary and these same findings to every other
6181                        // seat on the panel.
6182                        record.summary =
6183                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6184                        record.vote = Some(review.vote);
6185                        record.duration_ms = out.duration_ms;
6186                        for (n, mut f) in review.findings.into_iter().enumerate() {
6187                            // ids are magi's, never the agent's: the fixer's
6188                            // adoption report is keyed by them.
6189                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6190                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6191                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6192                            // `file` is agent-supplied prose too, never
6193                            // checked against the real tree — the same
6194                            // exposure `title`/`detail` above have, just in
6195                            // a field easy to forget because it looks like a
6196                            // path rather than free text.
6197                            f.file = f
6198                                .file
6199                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6200                            all_findings.push(f.clone());
6201                            record.findings.push(f);
6202                        }
6203                        self.state.event(
6204                            "review",
6205                            format!(
6206                                "round {round}: reviewer {} voted {} with {} finding(s)",
6207                                r + 1,
6208                                review.vote.label(),
6209                                record.findings.len()
6210                            ),
6211                        );
6212                    }
6213                    Err(e) => {
6214                        record.failed = Some(e.to_string());
6215                        self.state.event(
6216                            "review",
6217                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6218                        );
6219                    }
6220                }
6221                records.push(record);
6222            }
6223
6224            // Tally the round's votes and, if they split, spend the one
6225            // round of reconsideration the split -> deliberate -> revote
6226            // shape `judge`/`vote` use for the panel, sized down to what a
6227            // read-only review round can afford: one round, and a revote
6228            // rather than an argument, because the panel already wrote its
6229            // reasoning down as findings the first time around.
6230            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6231            let vote_split =
6232                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6233            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6234            if vote_split {
6235                self.state.event(
6236                    "review",
6237                    format!(
6238                        "round {round}: votes split ({}) — one round of reconsideration",
6239                        initial_votes
6240                            .iter()
6241                            .map(|v| v.label())
6242                            .collect::<Vec<_>>()
6243                            .join(", ")
6244                    ),
6245                );
6246                // Seats read every seat's findings and votes, still numbered
6247                // and never named — the same anonymity `review` itself keeps.
6248                let panel: Vec<ReviewSeatReport<'_>> = records
6249                    .iter()
6250                    .filter_map(|r| {
6251                        r.vote.map(|vote| ReviewSeatReport {
6252                            reviewer: r.reviewer,
6253                            vote,
6254                            summary: &r.summary,
6255                            findings: &r.findings,
6256                        })
6257                    })
6258                    .collect();
6259
6260                let mut jobs = Vec::new();
6261                let mut seats_at = Vec::new();
6262                for (r, spec) in reviewers.iter().cloned().enumerate() {
6263                    // A seat with no initial vote has nothing to reconsider
6264                    // from and stays absent, the same as it stayed absent
6265                    // from `panel` above.
6266                    if records[r].vote.is_none() {
6267                        continue;
6268                    }
6269                    let wt = root.join(format!("review-{}", r + 1));
6270                    let seat_key = format!("review-{}", r + 1);
6271                    let spec = self.occupant(&seat_key, spec);
6272                    let seat = self.seat(&seat_key, &spec.id);
6273                    // A seat with no live session has already forgotten the
6274                    // initial review's prompt — restate the patch it is
6275                    // voting on, the same as `deliberate`/`vote` do for a
6276                    // judge in the same position.
6277                    // The panel already carries this seat's own review and
6278                    // vote, so restating the patch makes the prompt whole for
6279                    // a seat handed to another agent.
6280                    let build = |with_patch: bool| {
6281                        prompt::review_reconsider(&ReviewReconsiderCtx {
6282                            instruction: &self.state.instruction,
6283                            reviewer: r + 1,
6284                            lens: Lens::for_seat(r),
6285                            panel: &panel,
6286                            patch: with_patch.then_some(ReviewPatch {
6287                                branch: &winner.branch,
6288                                base_short: &base_short,
6289                                stat: &stat,
6290                                patch: &patch,
6291                            }),
6292                            round,
6293                            rounds: max_rounds,
6294                            language: &language,
6295                        })
6296                    };
6297                    let full = build(true);
6298                    let prompt = if has_context(&spec, &seat, sessions) {
6299                        build(false)
6300                    } else {
6301                        full.clone()
6302                    };
6303                    jobs.push(SeatJob {
6304                        prompt,
6305                        spec,
6306                        seat,
6307                        cwd: wt,
6308                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6309                        allow_write: false,
6310                        sessions,
6311                        artifacts: artifacts.clone(),
6312                        stem: format!("review-{round}-reconsider-{}", r + 1),
6313                        handover: Some(full),
6314                    });
6315                    seats_at.push(r);
6316                }
6317
6318                let mut recon_quota_losses = Vec::new();
6319                let recon_cache = self.state.config.cache_dir();
6320                let recon_ctx = WaveCtx {
6321                    carry_seats: true,
6322                    pauses: Some((&self.pause, &self.interrupt)),
6323                    run: &run_id,
6324                    node: "review",
6325                    prompts: &prompts,
6326                    cache: recon_cache.as_deref(),
6327                    round: Some(round),
6328                };
6329                let recon_results = ask_json_wave::<ReviewRevote>(
6330                    jobs,
6331                    Arc::clone(&self.sem),
6332                    review_retries,
6333                    &self.roles.reviewer_roster,
6334                    &recon_ctx,
6335                    &mut recon_quota_losses,
6336                    &mut self.state,
6337                    &|_: &ReviewRevote| Ok(()),
6338                )
6339                .await;
6340                if self.state.resource_block.is_some() {
6341                    return self.stop_for_resource();
6342                }
6343                self.state.quota.extend(recon_quota_losses);
6344
6345                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6346                    let agent_id = seat.agent.clone();
6347                    self.state.seats.insert(seat.key.clone(), seat);
6348                    let mut rec = ReviewRevoteRecord {
6349                        reviewer: r + 1,
6350                        agent: agent_id,
6351                        vote: None,
6352                        reason: String::new(),
6353                        failed: None,
6354                    };
6355                    match res {
6356                        Ok((rv, _)) => {
6357                            rec.vote = Some(rv.vote);
6358                            rec.reason =
6359                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6360                            self.state.event(
6361                                "review",
6362                                format!(
6363                                    "round {round}: reviewer {} revoted {}",
6364                                    r + 1,
6365                                    rv.vote.label()
6366                                ),
6367                            );
6368                        }
6369                        Err(e) => {
6370                            rec.failed = Some(e.to_string());
6371                            self.state.event(
6372                                "review",
6373                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6374                            );
6375                        }
6376                    }
6377                    reconsideration.push(rec);
6378                }
6379            } else if initial_votes.len() > 1 {
6380                self.state.event(
6381                    "review",
6382                    format!(
6383                        "round {round}: votes agreed ({}) — no reconsideration",
6384                        initial_votes[0].label()
6385                    ),
6386                );
6387            }
6388
6389            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6390            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6391            // A round that already has a blocking finding and a round left to
6392            // try is going back to the fixer no matter what `verify.e2e`
6393            // says, so running it first only spends the loop's slowest step
6394            // (minutes, for a Rust repo's full test suite) on a head about
6395            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6396            // runs once a round has no blocking findings left (see
6397            // `round_is_clean`, which a deferred — empty — `e2e` can never
6398            // satisfy since `blocking` is nonzero whenever this branch is
6399            // taken), and `stop_reviewing` forces a real run before it will
6400            // ever read a deferred round as green.
6401            let defer_e2e =
6402                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6403            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6404                let reason =
6405                    format!("{blocking} blocking finding(s) already required a fix this round");
6406                self.state.event(
6407                    "verify",
6408                    format!(
6409                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6410                         {}); it will run once a round has none left",
6411                        short(&head)
6412                    ),
6413                );
6414                (Vec::new(), false, true, Some(reason))
6415            } else {
6416                let e2e_commands = self.state.config.verify.e2e.clone();
6417                let cache_dir = self.state.config.cache_dir();
6418                let context = format!("round {round}");
6419                let wait = self.repository_wait();
6420                let (e2e, verify_retried) = with_cache_lease(
6421                    &mut self.state,
6422                    cache_dir.as_deref(),
6423                    "e2e",
6424                    "e2e",
6425                    &winner.worktree,
6426                    &head,
6427                    verify_timeout,
6428                    &context,
6429                    &wait,
6430                    |state, budget| {
6431                        let shell = shell.clone();
6432                        let e2e_commands = e2e_commands.clone();
6433                        let worktree = winner.worktree.clone();
6434                        let context = context.clone();
6435                        async move {
6436                            run_e2e_with_retry(
6437                                state,
6438                                &shell,
6439                                &e2e_commands,
6440                                &worktree,
6441                                budget,
6442                                &context,
6443                            )
6444                            .await
6445                        }
6446                    },
6447                )
6448                .await?;
6449                (e2e, verify_retried, false, None)
6450            };
6451
6452            let expected = records.len();
6453            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6454            let incomplete = answered < expected;
6455            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6456            let policy = self.state.config.graph.incomplete_review;
6457            let clean = round_is_clean(
6458                blocking,
6459                e2e_ok,
6460                answered,
6461                expected,
6462                round_quota_missing,
6463                policy,
6464            );
6465
6466            let mut round_record = ReviewRound {
6467                round,
6468                head: head.clone(),
6469                verified_head: None,
6470                verified_at: None,
6471                reviews: records,
6472                e2e,
6473                verify_retried,
6474                e2e_deferred,
6475                e2e_defer_reason,
6476                fix: None,
6477                blocking,
6478                answered,
6479                expected,
6480                clean,
6481                progressed: false,
6482                vote_split,
6483                reconsideration,
6484                verdict: None,
6485            };
6486            // The final vote per seat is its revote where reconsideration
6487            // ran and answered, its initial vote otherwise — the same
6488            // fallback `tally` uses for a judge whose private vote failed.
6489            round_record.verdict = ReviewVote::worst(
6490                round_record
6491                    .final_votes()
6492                    .into_iter()
6493                    .map(|(_, _, vote)| vote),
6494            );
6495            // Which commit and when magi actually attempted to check —
6496            // known the moment a command was dispatched against `head`,
6497            // whether or not it finished: a resource-blocked attempt still
6498            // targeted a specific commit at a specific time, and leaving
6499            // that unrecorded is exactly what made `verification_summary`
6500            // report a fresh attempt as "commit unknown ... recorded before
6501            // this was tracked", indistinguishable from a genuinely old,
6502            // untracked record. Only a deferred or unconfigured round never
6503            // ran at all and has nothing to record — see
6504            // `ReviewRound::verified_head`'s own doc.
6505            if !matches!(
6506                round_record.e2e_status(),
6507                E2eStatus::Deferred | E2eStatus::NotConfigured
6508            ) {
6509                round_record.verified_head = Some(head.clone());
6510                round_record.verified_at = Some(Timestamp::now());
6511            }
6512
6513            if incomplete {
6514                let missing: Vec<String> = round_record
6515                    .reviews
6516                    .iter()
6517                    .filter(|r| r.failed.is_some())
6518                    .map(|r| format!("review-{}", r.reviewer))
6519                    .collect();
6520                self.state.event(
6521                    "review",
6522                    format!(
6523                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6524                        missing.join(", ")
6525                    ),
6526                );
6527            }
6528
6529            if clean {
6530                self.state.event(
6531                    "review",
6532                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6533                        format!(
6534                            "round {round}: clean (warn policy, incomplete panel) — no \
6535                             blocking findings from the seats that answered, verification green"
6536                        )
6537                    } else if incomplete {
6538                        format!(
6539                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6540                             quorum) — no blocking findings from the seats that answered, \
6541                             verification green",
6542                            expected - answered
6543                        )
6544                    } else {
6545                        format!("round {round}: clean — no blocking findings, verification green")
6546                    },
6547                );
6548                self.state.reviews.push(round_record);
6549                self.state.status = RunStatus::Gating;
6550                self.state.save()?;
6551                return Ok(());
6552            }
6553
6554            // Nothing was raised and verification passed, but not every seat
6555            // answered and `round_is_clean` still refused to call it clean —
6556            // either a seat is missing for a reason other than its own quota
6557            // (a crash, a timeout, unparsable output — worth another try), or
6558            // every seat that could have answered lost its quota and nobody
6559            // is left to decide on: re-review rather than send the fixer
6560            // after a round with nothing to fix.
6561            if incomplete && blocking == 0 && e2e_ok {
6562                self.state.reviews.push(round_record);
6563                self.state.save()?;
6564                if round == max_rounds {
6565                    self.state.status = RunStatus::Blocked;
6566                    self.state.event(
6567                        "review",
6568                        format!(
6569                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6570                             refusing to call it clean",
6571                            expected - answered
6572                        ),
6573                    );
6574                    return Ok(());
6575                }
6576                continue;
6577            }
6578
6579            // Nothing for the fixer to act on (`blocking == 0`) and the only
6580            // reason this round is not clean is that magi itself never got
6581            // a command to run — the shared build cache, not the patch (see
6582            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6583            // the fixer would invite a change to appease contention that has
6584            // nothing to do with the diff, and would leave this attempt
6585            // sitting in the next round's prompt as if it were about an
6586            // earlier, superseded commit rather than what it actually is:
6587            // the same head, still waiting to be checked. Wait for it the
6588            // same way the final round's own contention is already handled,
6589            // whatever round this happens to be.
6590            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6591                self.state.reviews.push(round_record);
6592                return self
6593                    .stop_unresolved_review(
6594                        "the round's own verification could not run",
6595                        &shell,
6596                        &winner.worktree,
6597                    )
6598                    .await;
6599            }
6600
6601            if round == max_rounds {
6602                self.state.reviews.push(round_record);
6603                return self
6604                    .stop_reviewing(
6605                        &format!(
6606                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6607                        ),
6608                        &shell,
6609                        &winner.worktree,
6610                    )
6611                    .await;
6612            }
6613
6614            if !self.fix_review_round(&winner, round_record).await? {
6615                return Ok(());
6616            }
6617        }
6618        Ok(())
6619    }
6620
6621    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6622    /// False means admission or continuation policy stopped the review loop.
6623    async fn fix_review_round(
6624        &mut self,
6625        winner: &Candidate,
6626        mut round_record: ReviewRound,
6627    ) -> Result<bool> {
6628        let round = round_record.round;
6629        let shell = self.state.config.shell();
6630        let step = self.run_fix_round(winner, &mut round_record).await?;
6631        if matches!(step, FixStep::Resource) {
6632            self.stop_for_resource()?;
6633            return Ok(false);
6634        }
6635        if self.state.reviews.len() < round {
6636            self.state.reviews.push(round_record);
6637        } else {
6638            self.state.reviews[round - 1] = round_record;
6639        }
6640        self.state.save()?;
6641
6642        // The fixer's own report never came back this round, even after
6643        // `continue_fix_report`'s own budget was spent on it — not an
6644        // ordinary "no report" (dropped stream, quota, plain failure),
6645        // which already reads that way and is left to the existing round
6646        // budget. Stopping here, rather than opening another round, is
6647        // what keeps a next reviewer/fixer wave from ever being
6648        // dispatched onto `winner.worktree` while whatever the seat's
6649        // last call may still have running there is unaccounted for: no
6650        // process liveness check exists (and none is being added — see
6651        // AGENTS.md/this task's own scope), so the only way to honour
6652        // "nothing starts before a valid report returns" is to not start
6653        // anything further on this worktree from this run at all.
6654        if matches!(step, FixStep::Recorded { report_lost: true }) {
6655            self.stop_reviewing(
6656                "the fixer's adoption report never came back, even after resuming its \
6657                         own seat; refusing to start another round against the same worktree \
6658                         while that is unresolved",
6659                &shell,
6660                &winner.worktree,
6661            )
6662            .await?;
6663            return Ok(false);
6664        }
6665
6666        let streak = self
6667            .state
6668            .reviews
6669            .iter()
6670            .rev()
6671            .take_while(|r| !r.progressed)
6672            .count();
6673        if streak >= STAGNANT_LIMIT {
6674            self.stop_reviewing(
6675                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6676                &shell,
6677                &winner.worktree,
6678            )
6679            .await?;
6680            return Ok(false);
6681        }
6682        Ok(true)
6683    }
6684
6685    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6686    async fn run_fix_round(
6687        &mut self,
6688        winner: &Candidate,
6689        round_record: &mut ReviewRound,
6690    ) -> Result<FixStep> {
6691        let round = round_record.round;
6692        let max_rounds = self.state.config.graph.review_rounds;
6693        let run_id = self.state.id.clone();
6694        let prompts = self.state.config.prompts.clone();
6695        let language = self.state.config.graph.language.clone();
6696        let sessions = self.state.config.graph.sessions;
6697        let artifacts = self.merge_repair_artifacts();
6698        let base = self.landing_base();
6699        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6700        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6701        let this_round_verification = round_record.verification_summary(&head);
6702        let all_findings: Vec<_> = round_record
6703            .reviews
6704            .iter()
6705            .flat_map(|r| r.findings.iter().cloned())
6706            .collect();
6707        // Fix. The winner's own implementer seat continues its conversation:
6708        // the competition is over, so context is pure benefit now.
6709        let blocking_findings: Vec<_> = all_findings
6710            .iter()
6711            .filter(|f| f.severity.blocks())
6712            .cloned()
6713            .collect();
6714        let fix_prompt = prompt::fix_in(
6715            &self.state.instruction,
6716            &blocking_findings,
6717            this_round_verification.as_ref(),
6718            round,
6719            max_rounds,
6720            &language,
6721            &self.state.config.graph.github_language,
6722        );
6723        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6724        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6725        let (job, seat, out) = self
6726            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6727                prompt: fix_prompt.clone(),
6728                spec,
6729                seat,
6730                cwd: winner.worktree.clone(),
6731                timeout,
6732                allow_write: true,
6733                sessions,
6734                artifacts: artifacts.clone(),
6735                stem: format!("fix-{round}"),
6736                handover: None,
6737            })
6738            .await;
6739        // Not asked: the round is not recorded (so a resume repeats it),
6740        // no fixer was handed over to, and the seat is left as it was.
6741        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6742            return Ok(FixStep::Resource);
6743        }
6744        let agent_id = seat.agent.clone();
6745
6746        let mut fix = FixRecord {
6747            agent: agent_id,
6748            addressed: Vec::new(),
6749            rejected: Vec::new(),
6750            notes: String::new(),
6751            committed: false,
6752            failed: None,
6753            duration_ms: 0,
6754            continuation: None,
6755        };
6756        let mut continuation = ContinuationRecord::not_needed();
6757        let mut final_seat = seat.clone();
6758        match out {
6759            AgentOutcome::Ok(o) => {
6760                fix.duration_ms = o.duration_ms;
6761                let parsed = verdict::extract_json::<FixReport>(&o.text);
6762                // A parsed report standing next to a command this same
6763                // reply's own CLI never confirmed the exit status of is
6764                // not a resolved answer — the identical `CommandEvidence`
6765                // `state.jobs` renders, read here instead of only on
6766                // display, per the completion judgment and the shown
6767                // record needing to agree.
6768                let incomplete_reason = match &parsed {
6769                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6770                        "the reply parsed, but it reported a command whose own CLI never \
6771                             confirmed an exit status"
6772                            .to_owned(),
6773                    ),
6774                    Ok(_) => None,
6775                    Err(e) => Some(e.to_string()),
6776                };
6777                match incomplete_reason {
6778                    None => {
6779                        let report = parsed.expect("checked Ok above");
6780                        fix.addressed = report.addressed;
6781                        fix.rejected = report.rejected;
6782                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6783                    }
6784                    Some(reason) => {
6785                        let (resumed_seat, resolved, failure, cont) = self
6786                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6787                            .await;
6788                        fix.duration_ms += cont.cumulative_wait_ms;
6789                        continuation = cont;
6790                        final_seat = resumed_seat;
6791                        match resolved {
6792                            Some(report) => {
6793                                fix.addressed = report.addressed;
6794                                fix.rejected = report.rejected;
6795                                fix.notes =
6796                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6797                            }
6798                            None => fix.failed = failure,
6799                        }
6800                    }
6801                }
6802            }
6803            // The CLI's raw error JSON is not a fix report to parse.
6804            AgentOutcome::Dropped(o) => {
6805                fix.duration_ms = o.duration_ms;
6806                let why = o
6807                    .dropped
6808                    .as_ref()
6809                    .map(|d| d.why.as_str())
6810                    .unwrap_or("the CLI ended the stream without delivering its answer");
6811                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6812            }
6813            AgentOutcome::Quota(o) => {
6814                self.state.quota.push(QuotaLoss {
6815                    seat: final_seat.key.clone(),
6816                    node: "fix".to_owned(),
6817                    at: Timestamp::now(),
6818                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6819                });
6820                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6821            }
6822            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6823        }
6824        fix.continuation = Some(continuation);
6825        self.state.seats.insert(final_seat.key.clone(), final_seat);
6826        let coordination = self.coordinate(&winner.worktree).await?;
6827        if let Ok(r) = git::rescue_commit(
6828            &winner.worktree,
6829            &format!("magi: review round {round} fixes (uncommitted work)"),
6830        )
6831        .await
6832        {
6833            self.state.note_withheld("fix", &r.withheld);
6834        }
6835        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6836        fix.committed = after != before;
6837        // Judged by what `git` says moved against base, never by the
6838        // fixer's own `addressed`/`rejected` count — see
6839        // `ReviewRound::progressed`. Propagated with `?`, the same as the
6840        // `patch` snapshot above: swallowing this error would default
6841        // `diff_after` to empty, which almost always differs from a
6842        // non-empty `patch` and reads as "progressed" — exactly backwards
6843        // for a `git` failure the stagnation check cannot see through.
6844        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6845        drop(coordination);
6846        let progressed = diff_after != patch;
6847        let commit_note = if fix.committed {
6848            "committed"
6849        } else {
6850            "NO new commit"
6851        };
6852        let tree_note = if progressed {
6853            "changed vs base"
6854        } else {
6855            "unchanged vs base"
6856        };
6857        self.state.event(
6858            "fix",
6859            match &fix.failed {
6860                // Distinct on purpose from "0 addressed, 0 rejected": the
6861                // fixer's own diff still landed (blocking counts do keep
6862                // falling round over round), only its adoption report did
6863                // not come back, so this must never read like every
6864                // finding was reviewed and declined.
6865                Some(reason) => {
6866                    format!(
6867                        "round {round}: fixer's adoption report was lost ({reason}); \
6868                             {commit_note}, tree {tree_note}"
6869                    )
6870                }
6871                None => format!(
6872                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6873                         {tree_note}{}",
6874                    fix.addressed.len(),
6875                    fix.rejected.len(),
6876                    if continuation.outcome == ContinuationOutcome::Resumed {
6877                        format!(
6878                            " (adoption report recovered after {} continuation(s))",
6879                            continuation.attempts
6880                        )
6881                    } else {
6882                        String::new()
6883                    },
6884                ),
6885            },
6886        );
6887        round_record.fix = Some(fix);
6888        round_record.progressed = progressed;
6889        Ok(FixStep::Recorded {
6890            report_lost: matches!(
6891                continuation.outcome,
6892                ContinuationOutcome::Exhausted
6893                    | ContinuationOutcome::QuotaLost
6894                    | ContinuationOutcome::NoSession
6895            ),
6896        })
6897    }
6898
6899    /// Decide, from the last recorded round's own verification, whether
6900    /// stopping the review loop is a hand-off or a genuine block.
6901    ///
6902    /// Called once the loop has given up trying — the round budget is spent,
6903    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6904    /// findings still open, never while a round is still clean or the
6905    /// incomplete-panel case handled inline above. Gate and e2e are facts
6906    /// about the tree; a lingering review finding is an opinion, and this
6907    /// workload's own `magi stats` puts reviewer precision low enough
6908    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6909    /// must not by itself stand between a green, verified change and the
6910    /// human who decides what to do with it. A red e2e is not an opinion, so
6911    /// that case still blocks, with the failing command and a tail of its
6912    /// output recorded here rather than left in `run.json` for someone to go
6913    /// find.
6914    ///
6915    /// A round that deferred its own e2e (see [`Config::graph`]'s
6916    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6917    /// only because nothing ran, and treating an empty list as a passing one
6918    /// here is exactly the "deferred painted green" bug this function exists
6919    /// to not have. When the last round's own verification never resolved —
6920    /// deferred on purpose, or a real attempt the shared build cache blocked
6921    /// — this makes (or retries) the real run, on the actual worktree this
6922    /// loop is about to stop touching, before deciding anything. A
6923    /// resource-blocked attempt is likewise never read as either green or
6924    /// red: it is evidence about the machine, not the patch (see
6925    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6926    /// blocked cache leaves this call without deciding rather than guessing
6927    /// — the caller retries on a later reentry.
6928    /// Record, once, that the review loop handed off over a blocking finding
6929    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6930    /// `land` asks the owner even with `land_approval` off. Called from every
6931    /// path that concludes `Gating`; a reentry keeps the first record.
6932    fn record_contested_handoff(&mut self) {
6933        if self.state.contested_handoff.is_some() {
6934            return;
6935        }
6936        let Some(contested) = self
6937            .state
6938            .reviews
6939            .last()
6940            .and_then(ReviewRound::contested_handoff)
6941        else {
6942            return;
6943        };
6944        self.state.event(
6945            "review",
6946            format!(
6947                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6948                 wait for the owner's approval",
6949                contested.findings.len(),
6950                contested.rejecters.len()
6951            ),
6952        );
6953        self.state.contested_handoff = Some(contested);
6954    }
6955
6956    async fn stop_unresolved_review(
6957        &mut self,
6958        why: &str,
6959        shell: &[String],
6960        worktree: &Path,
6961    ) -> Result<()> {
6962        self.stop_reviewing_inner(why, shell, worktree, true).await
6963    }
6964
6965    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6966        self.stop_reviewing_inner(why, shell, worktree, false).await
6967    }
6968
6969    async fn stop_reviewing_inner(
6970        &mut self,
6971        why: &str,
6972        shell: &[String],
6973        worktree: &Path,
6974        repairable: bool,
6975    ) -> Result<()> {
6976        let round_idx = self.state.reviews.len() - 1;
6977        // A deferred round and a resource-blocked one are the same shape
6978        // here: neither has a real result yet, and both get one more
6979        // attempt. Read off `e2e_status` — the single source for this —
6980        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6981        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6982        // still retries instead of being read as a settled result the
6983        // instant it stops being empty.
6984        let needs_catchup_run = matches!(
6985            self.state.reviews[round_idx].e2e_status(),
6986            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6987        );
6988        if needs_catchup_run {
6989            let round = self.state.reviews[round_idx].round;
6990            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6991            let commands = self.state.config.verify.e2e.clone();
6992            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6993            let cache_dir = self.state.config.cache_dir();
6994            let context = format!(
6995                "round {round}: verification unresolved, catching up before the final decision"
6996            );
6997            let wait = self.repository_wait();
6998            let (outcomes, verify_retried) = with_cache_lease(
6999                &mut self.state,
7000                cache_dir.as_deref(),
7001                "e2e",
7002                "e2e",
7003                worktree,
7004                &attempted_head,
7005                timeout,
7006                &context,
7007                &wait,
7008                |state, budget| {
7009                    let shell = shell.to_vec();
7010                    let commands = commands.clone();
7011                    let context = context.clone();
7012                    async move {
7013                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7014                            .await
7015                    }
7016                },
7017            )
7018            .await?;
7019            let last = &mut self.state.reviews[round_idx];
7020            last.e2e = outcomes;
7021            last.verify_retried = verify_retried;
7022            // Always the commit and time this attempt actually targeted,
7023            // whether or not it happens to equal the reviewed `head` and
7024            // whether or not a command finished — see
7025            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7026            // attempt is recorded too, so a later reader sees "attempted
7027            // again at T2" rather than silence.
7028            last.verified_head = Some(attempted_head);
7029            last.verified_at = Some(Timestamp::now());
7030            if verify_inconclusive(&last.e2e) {
7031                // Still not a real result: `e2e_deferred` is left exactly
7032                // as it was, so `needs_catchup_run` above reads
7033                // `ResourceBlocked` (via `e2e_status`, which checks
7034                // `resource_blocked` before `e2e_deferred`) and retries
7035                // again on the next reentry, rather than recording
7036                // contention as a red e2e and blocking the run on it.
7037                self.state.save()?;
7038                return Ok(());
7039            }
7040            last.e2e_deferred = false;
7041        }
7042        let last = &self.state.reviews[round_idx];
7043        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7044
7045        match last.e2e_status() {
7046            E2eStatus::Failed => {
7047                let red: Vec<String> = last
7048                    .e2e
7049                    .iter()
7050                    .filter(|o| !o.ok())
7051                    .map(|o| {
7052                        format!(
7053                            "`{}` -> {:?}\n{}",
7054                            o.command,
7055                            o.code,
7056                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7057                        )
7058                    })
7059                    .collect();
7060                let can_repair = last.blocking == 0 && last.fix.is_none();
7061                self.state.reviews[round_idx].clean = false;
7062                self.state
7063                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7064                self.state.status = RunStatus::Blocked;
7065                if repairable
7066                    && can_repair
7067                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7068                {
7069                    self.state.status = RunStatus::Reviewing;
7070                }
7071            }
7072            // `needs_catchup_run` above already retried once this call; if
7073            // it is still blocked, this is magi's own admission it could
7074            // not get a command to run, never a verdict on the patch — the
7075            // run is left exactly where a later reentry can retry again.
7076            E2eStatus::ResourceBlocked => {
7077                self.state.event(
7078                    "review",
7079                    format!(
7080                        "{why}; e2e could not run (resource unavailable); not \
7081                         deciding yet"
7082                    ),
7083                );
7084            }
7085            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7086                self.state.event(
7087                    "review",
7088                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7089                );
7090                self.record_contested_handoff();
7091                self.state.status = RunStatus::Gating;
7092            }
7093        }
7094        self.state.save()?;
7095        Ok(())
7096    }
7097
7098    // ----------------------------------------------------------------- gate
7099
7100    async fn gate(&mut self) -> Result<()> {
7101        // Judged by the review record itself, not by `status`: a solo
7102        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7103        // reentry (see `judge`), and trusting it here is exactly how a run
7104        // that exhausted its review budget got gated and merged a second
7105        // time around. `review_conclusion` recomputes the review loop's own
7106        // verdict from the round records themselves — `Gating` for a clean
7107        // round or a hand-off (see `stop_reviewing`), anything else means the
7108        // loop is still going or genuinely blocked.
7109        // A base the winner could not be replayed onto is a decision, not a
7110        // round: there is no landing tree to gate. Read as its own record for
7111        // the same reason the review verdict is.
7112        if self.state.status == RunStatus::Failed
7113            || self
7114                .state
7115                .base_sync
7116                .as_ref()
7117                .is_some_and(|s| s.conflict.is_some())
7118            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7119                != Some(RunStatus::Gating)
7120        {
7121            return Ok(());
7122        }
7123        if self.state.gate_ran {
7124            // `review_loop` derives its conclusion from the clean review
7125            // record on every reentry and therefore puts a completed run back
7126            // in `Gating`. A recorded gate is a stronger, terminal fact:
7127            // retain its original command output (or lack of any, for a repo
7128            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7129            // doc) and restore `Blocked` on a real failure rather than
7130            // pretending the command is still running or running it a second
7131            // time. `gate_ran == false` remains the only shape — unattempted,
7132            // or a resource-blocked retry — that may still need to execute a
7133            // command.
7134            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7135                self.state.status = RunStatus::Blocked;
7136                self.state.save()?;
7137            }
7138            return Ok(());
7139        }
7140        let Some(winner) = self.state.winner().cloned() else {
7141            return Ok(());
7142        };
7143        self.state.status = RunStatus::Gating;
7144        let mut outcomes = self.run_gate(&winner).await?;
7145        loop {
7146            // A resource-blocked outcome means the gate command never actually
7147            // ran - the shared build cache could not be acquired or confirmed
7148            // fresh in time - which is evidence about the machine, not about
7149            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7150            // Recording it as a red gate would mark a run `Blocked` on nothing
7151            // but contention magi has already logged; leaving `self.state.gate`
7152            // empty and `self.state.gate_ran` false instead keeps the shape
7153            // this function already treats as "still needs to run" (see the
7154            // early-return above), so the next call retries the command
7155            // rather than concluding anything.
7156            if verify_inconclusive(&outcomes) {
7157                self.state.save()?;
7158                return Ok(());
7159            }
7160            if outcomes.iter().all(CommandOutcome::ok) {
7161                break;
7162            }
7163            match self.gate_fix_round(&winner, &outcomes).await? {
7164                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7165                GateFix::Stop => break,
7166                GateFix::Defer => {
7167                    self.state.save()?;
7168                    return Ok(());
7169                }
7170            }
7171        }
7172        let passed = outcomes.iter().all(CommandOutcome::ok);
7173        self.state.gate = outcomes;
7174        self.state.gate_ran = true;
7175        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7176        if !passed {
7177            self.state.status = RunStatus::Blocked;
7178            let spent = self.state.gate_fixes.len();
7179            self.state.event(
7180                "gate",
7181                if spent == 0 {
7182                    "gate failed; not merging".to_owned()
7183                } else {
7184                    format!("gate failed after {spent} gate-fix round(s); not merging")
7185                },
7186            );
7187        }
7188        self.state.save()?;
7189        Ok(())
7190    }
7191
7192    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7193    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7194    /// after review is clean and never on a candidate awaiting judging.
7195    ///
7196    /// Never fails the run: a non-zero exit or timeout is a warning and a
7197    /// recorded outcome, and the gate remains the single arbiter. Nothing
7198    /// configured means nothing happens - no event, no commit. `commit_all`
7199    /// commits any leftover change under the neutral identity and returns
7200    /// `false` when the tree is clean, so no empty commit is ever made.
7201    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7202        let commands = state.config.verify.pre_gate.clone();
7203        if commands.is_empty() {
7204            return;
7205        }
7206        let shell = state.config.shell();
7207        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7208        let (outcomes, _) = run_commands(
7209            state,
7210            "pre_gate",
7211            "pre_gate",
7212            0,
7213            &shell,
7214            &commands,
7215            &winner.worktree,
7216            timeout,
7217        )
7218        .await;
7219        for o in &outcomes {
7220            if !o.ok() {
7221                tracing::warn!(
7222                    "pre_gate `{}` failed ({:?}); the gate decides",
7223                    o.command,
7224                    o.code
7225                );
7226            }
7227            state.event(
7228                "pre_gate",
7229                format!(
7230                    "`{}` -> {}",
7231                    o.command,
7232                    if o.ok() {
7233                        "pass".to_owned()
7234                    } else {
7235                        format!(
7236                            "FAIL ({:?})\n{}",
7237                            o.code,
7238                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7239                        )
7240                    }
7241                ),
7242            );
7243        }
7244        state.pre_gate = outcomes;
7245        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7246            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7247                Ok(head) => {
7248                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7249                    state.pre_gate_commit = Some(head);
7250                }
7251                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7252            },
7253            Ok(false) => {}
7254            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7255        }
7256        if let Err(e) = state.save() {
7257            tracing::warn!("could not persist the pre_gate record: {e:#}");
7258        }
7259    }
7260
7261    /// Run `verify.gate` once against the winner's current tree, logging one
7262    /// event per command. Empty when nothing is configured.
7263    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7264        let wait = self.repository_wait();
7265        run_landing_gate(&mut self.state, winner, &wait).await
7266    }
7267
7268    /// One bounded fix round for a failing gate.
7269    ///
7270    /// The fixer is told the failure came from the gate itself, not from a
7271    /// reviewer, and is shown the failed commands, their exit codes and a tail
7272    /// of their output - whatever `[verify].gate` holds, nothing here knows
7273    /// what those commands run. Only a normal non-zero exit that printed
7274    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7275    /// command or a full disk says nothing about the code, and a fixer sent
7276    /// after it can only appease the machine. The round is judged by what git
7277    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7278    /// again before the gate does, so a fix cannot trade a green gate for a
7279    /// red e2e unnoticed.
7280    async fn gate_fix_round(
7281        &mut self,
7282        winner: &Candidate,
7283        outcomes: &[CommandOutcome],
7284    ) -> Result<GateFix> {
7285        let cap = self.state.config.graph.gate_fix_rounds;
7286        let spent = self.state.gate_fixes.len();
7287        if spent >= cap {
7288            if cap > 0 {
7289                self.state.event(
7290                    "gate",
7291                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7292                );
7293            }
7294            return Ok(GateFix::Stop);
7295        }
7296        if !gate_fixable(outcomes) {
7297            self.state.event(
7298                "gate",
7299                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7300                 command or similar); not spending a fix round on it",
7301            );
7302            return Ok(GateFix::Stop);
7303        }
7304        let min_free = self.state.config.disk.min_free_bytes;
7305        if min_free > 0 {
7306            match crate::disk::free_bytes(&winner.worktree) {
7307                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7308                Ok(free) => {
7309                    self.state.event(
7310                        "gate",
7311                        format!(
7312                            "only {free} bytes free ({min_free} required by `[disk] \
7313                             min_free_bytes`); not spending a fix round on a failure the disk \
7314                             may explain"
7315                        ),
7316                    );
7317                    return Ok(GateFix::Stop);
7318                }
7319                Err(e) => {
7320                    self.state.event(
7321                        "gate",
7322                        format!("free disk space could not be measured ({e:#}); no fix round"),
7323                    );
7324                    return Ok(GateFix::Stop);
7325                }
7326            }
7327        }
7328
7329        let attempt = spent + 1;
7330        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7331        let base = self.landing_base();
7332        let fix_prompt = prompt::gate_fix_in(
7333            &self.state.instruction,
7334            &failed,
7335            attempt,
7336            cap,
7337            &self.state.config.graph.language,
7338            &self.state.config.graph.github_language,
7339        );
7340        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7341        let sessions = self.state.config.graph.sessions;
7342        let artifacts = self.merge_repair_artifacts();
7343        self.state.event(
7344            "gate",
7345            format!("gate failed; gate-fix round {attempt} of {cap}"),
7346        );
7347        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7348        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7349        // Persist before invoking the fixer: it may commit and then be
7350        // interrupted before its result (or the following e2e) is saved.
7351        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7352            head: before.clone(),
7353            passed: false,
7354            outcomes: Vec::new(),
7355        });
7356        self.state.save()?;
7357        let (_, seat, out) = self
7358            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7359                prompt: fix_prompt.clone(),
7360                spec,
7361                seat,
7362                cwd: winner.worktree.clone(),
7363                timeout,
7364                allow_write: true,
7365                sessions,
7366                artifacts: artifacts.clone(),
7367                stem: format!("gate-fix-{attempt}"),
7368                handover: None,
7369            })
7370            .await;
7371        // The fixer was never asked: no round is spent (nothing is recorded)
7372        // and a later reentry asks the same agent again.
7373        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7374            return Ok(GateFix::Defer);
7375        }
7376        let mut record = GateFixRecord {
7377            agent: seat.agent.clone(),
7378            failed,
7379            notes: String::new(),
7380            committed: false,
7381            error: None,
7382        };
7383        match out {
7384            AgentOutcome::Ok(o) => {
7385                // A missing report is not a failed fix: the round is judged
7386                // by the tree below, and the report only carries prose.
7387                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7388                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7389                }
7390            }
7391            AgentOutcome::Dropped(_) => {
7392                record.error = Some("the CLI dropped the stream".to_owned());
7393            }
7394            AgentOutcome::Quota(o) => {
7395                self.state.quota.push(QuotaLoss {
7396                    seat: seat.key.clone(),
7397                    node: "gate-fix".to_owned(),
7398                    at: Timestamp::now(),
7399                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7400                });
7401                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7402            }
7403            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7404        }
7405        self.state.seats.insert(seat.key.clone(), seat);
7406        let coordination = self.coordinate(&winner.worktree).await?;
7407        if let Ok(r) = git::rescue_commit(
7408            &winner.worktree,
7409            &format!("magi: gate fix {attempt} (uncommitted work)"),
7410        )
7411        .await
7412        {
7413            self.state.note_withheld("gate-fix", &r.withheld);
7414        }
7415        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7416        record.committed = after != before;
7417        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7418        let note = record.error.clone();
7419        self.state.gate_fixes.push(record);
7420        self.state.save()?;
7421        if !changed {
7422            self.state.event(
7423                "gate",
7424                match note {
7425                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7426                    None => format!("gate-fix round {attempt}: the tree did not change"),
7427                },
7428            );
7429            return Ok(GateFix::Stop);
7430        }
7431        self.state.event(
7432            "gate",
7433            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7434        );
7435
7436        drop(coordination);
7437        let wait = self.repository_wait();
7438        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7439    }
7440
7441    // ---------------------------------------------------------------- merge
7442
7443    /// One read-only rewrite by the summary's author; a text the gate still
7444    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7445    /// applies on their word or on silence. `None` means the run is parked on
7446    /// that question and nothing may be pushed or opened yet.
7447    async fn guarded_pr_message(
7448        &mut self,
7449        winner: &Candidate,
7450        facts: Option<&BranchFacts>,
7451        posting: bool,
7452    ) -> Result<Option<PrMessage>> {
7453        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7454        let head = match final_head.as_deref() {
7455            Some(h) => crate::pr_outcome::FinalHead::At(h),
7456            None => crate::pr_outcome::FinalHead::Unknown,
7457        };
7458        // Only a pull request that is about to be posted pays for a summary.
7459        let summarized = if posting {
7460            Some(self.summarize_change(winner, final_head.as_deref()).await)
7461        } else {
7462            None
7463        };
7464        // A busy shared resource is not a permanent cause: publishing an
7465        // "unavailable" description would stick (a later head move never
7466        // edits the body). Stop before anything is pushed or opened; the
7467        // run is `Blocked` and a resume asks the summary again.
7468        if self.state.resource_block.is_some() {
7469            self.state.event(
7470                "pr-summary",
7471                "a shared resource was busy; nothing was published, resume to summarize again",
7472            );
7473            return Ok(None);
7474        }
7475        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7476            anyhow::bail!(
7477                "the final commit moved during public description generation; retry before publishing"
7478            );
7479        }
7480        let note = match (&summarized, &self.state.change_summary) {
7481            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7482            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7483            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7484            (None, _) => ChangeNote::NotAsked,
7485        };
7486        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7487        if !posting {
7488            let identity = crate::scrub::Identity::current();
7489            return Ok(Some(PrMessage {
7490                title: crate::scrub::scrub(&pr.title, &identity),
7491                body: crate::scrub::scrub(&pr.body, &identity),
7492            }));
7493        }
7494        // This very text was already rejected and asked about: no second
7495        // judge call, rewrite or question, just carry the record on.
7496        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7497        if let Some(known) = self
7498            .state
7499            .github_text
7500            .clone()
7501            .filter(|g| g.fingerprint == raw_fp)
7502        {
7503            return self.settle_withheld(winner, &pr, known).await;
7504        }
7505        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7506        let mut violations =
7507            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7508        let identity = crate::scrub::Identity::current();
7509        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7510            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7511            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7512        {
7513            violations.push(crate::github_text::Violation::SensitiveData);
7514        }
7515        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7516        // only a language violation starts one; sensitive data alone goes
7517        // straight to the owner, with the redacted text as the fallback.
7518        let has_language = violations
7519            .iter()
7520            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7521        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7522            self.state.event(
7523                "github-text",
7524                if has_language {
7525                    format!("description rejected: {violations:?}; requesting one rewrite")
7526                } else {
7527                    format!("description rejected: {violations:?}; asking the owner")
7528                },
7529            );
7530            let mut rewritten = false;
7531            let mut rejected_rewrite = None;
7532            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7533            {
7534                let output = text;
7535                if let Some(title) = summary_title(&output) {
7536                    // Judge the redacted candidate that will actually be posted.
7537                    // Sensitive data alone must not discard a useful rewrite.
7538                    let raw_body = summary_without_title(&output);
7539                    let clean_title = crate::scrub::scrub(&title, &identity);
7540                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7541                    let redacted = clean_title != title || body != raw_body;
7542                    let title = clean_title;
7543                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7544                    if !complete {
7545                        self.state.event(
7546                            "github-text",
7547                            "description rewrite omitted required findings",
7548                        );
7549                    }
7550                    if !body.trim().is_empty() {
7551                        body.push_str("\n\n");
7552                        body.push_str(&public_pr_evidence(&self.state, head));
7553                    }
7554                    let marker = format!("magi:run/{}", self.state.id);
7555                    if !body.trim().is_empty() && !body.contains(&marker) {
7556                        body.push_str(&format!("\n\n{marker}"));
7557                    }
7558                    let rewrite = PrMessage {
7559                        title: title.clone(),
7560                        body: body.clone(),
7561                    };
7562                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7563                    let mut failed = crate::github_text::check_for(
7564                        &self.state.config.graph,
7565                        &title,
7566                        &body,
7567                        redecision,
7568                    );
7569                    if (!crate::github_text::shareable(&title)
7570                        || !crate::github_text::shareable(&body))
7571                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7572                    {
7573                        failed.push(crate::github_text::Violation::SensitiveData);
7574                    }
7575                    if complete
7576                        && !title.trim().is_empty()
7577                        && !body.trim().is_empty()
7578                        && !failed.is_empty()
7579                    {
7580                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7581                            title: title.clone(),
7582                            body: body.clone(),
7583                            failed: failed
7584                                .iter()
7585                                .map(|v| crate::github_text::category(*v).to_owned())
7586                                .collect(),
7587                        });
7588                    }
7589                    if complete
7590                        && !title.trim().is_empty()
7591                        && !raw_body.trim().is_empty()
7592                        && failed.is_empty()
7593                    {
7594                        decision = redecision;
7595                        pr = PrMessage { title, body };
7596                        rewritten = true;
7597                        self.state
7598                            .event("github-text", "description rewrite passed");
7599                        if redacted {
7600                            self.state.event(
7601                                "github-text",
7602                                "sensitive data removed from description rewrite",
7603                            );
7604                        }
7605                    }
7606                }
7607            }
7608            if !rewritten {
7609                self.state.event(
7610                    "github-text",
7611                    "description rewrite unavailable or rejected; asking the owner",
7612                );
7613                if let Some(w) =
7614                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7615                {
7616                    // A different text than the one asked about before: that
7617                    // question no longer describes anything.
7618                    self.retire_github_text_question("the withheld text changed");
7619                    let record = crate::run::GithubTextAsk {
7620                        rewrite: rejected_rewrite,
7621                        fingerprint: raw_fp,
7622                        title: w.title,
7623                        body: w.body,
7624                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7625                        question: None,
7626                        asks: 0,
7627                        resolved: false,
7628                        chosen_title: None,
7629                        choice: None,
7630                    };
7631                    return self.settle_withheld(winner, &pr, record).await;
7632                }
7633            }
7634        }
7635        let (title, body) =
7636            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7637        Ok(Some(PrMessage { title, body }))
7638    }
7639
7640    /// Describe the whole change from its final diff in either run mode.
7641    ///
7642    /// The summary is tied to the final commit and to the merge base of the
7643    /// resolved remote base; a record for any other pair is dropped and the
7644    /// agent asked again, so a description never outlives the diff it was
7645    /// written from. The seat is read-only and the prompt tells it to read the
7646    /// diff and the code around it. `[roles] synthesizer` selects the agents
7647    /// (unset: the default pick), each asked at most once, advancing on an
7648    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7649    /// carries the reason it is unavailable; nothing is invented then.
7650    async fn summarize_change(
7651        &mut self,
7652        winner: &Candidate,
7653        head: Option<&str>,
7654    ) -> std::result::Result<(), String> {
7655        let Some(head) = head else {
7656            return Err("the final commit could not be read".to_owned());
7657        };
7658        let remote = self.state.config.merge.remote.clone();
7659        let tracking = format!("{remote}/{}", self.state.base_branch);
7660        if !git::rev_exists(&winner.worktree, &tracking).await {
7661            return Err(format!("the remote base `{tracking}` could not be read"));
7662        }
7663        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7664            Ok(b) if !b.is_empty() => b,
7665            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7666        };
7667        let evidence = crate::pr_outcome::summary_evidence(
7668            &self.state,
7669            crate::pr_outcome::FinalHead::At(head),
7670        );
7671        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7672        if self
7673            .state
7674            .change_summary
7675            .as_ref()
7676            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7677        {
7678            return Ok(());
7679        }
7680        if self.state.change_summary.take().is_some() {
7681            self.state.event(
7682                "pr-summary",
7683                "the final diff or evidence changed; the earlier change summary was discarded",
7684            );
7685        }
7686        // Save the source snapshot even if no generation agent is available.
7687        if let Err(e) = write_artifact(
7688            &self.state,
7689            &format!("public-description-{fingerprint}.md"),
7690            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7691        ) {
7692            tracing::warn!("could not save public description evidence: {e:#}");
7693        }
7694        let chain = match agent::pick_chain(
7695            &self.state.config.agents,
7696            self.state.config.roles.synthesizer.as_ref(),
7697            &agent::installed,
7698            "synthesizer",
7699        ) {
7700            Ok(chain) => chain,
7701            Err(e) => {
7702                tracing::warn!("no agent for the change summary: {e:#}");
7703                self.state.event("pr-summary", "no summary agent available");
7704                return Err("no agent was available to summarize it".to_owned());
7705            }
7706        };
7707        let subjects = git::subjects(&winner.worktree, &base, head)
7708            .await
7709            .unwrap_or_default();
7710        let stat = git::diff_stat(&winner.worktree, &base, head)
7711            .await
7712            .unwrap_or_default();
7713        let stat: String = stat.chars().take(6_000).collect();
7714        let mut prompt = crate::prompt::change_summary_in(
7715            &base,
7716            head,
7717            &subjects,
7718            stat.trim(),
7719            &self.state.config.graph.github_language,
7720        );
7721        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7722        let sessions = self.state.config.graph.sessions;
7723        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7724        let prompts = self.state.config.prompts.clone();
7725        let cache = self.state.config.cache_dir();
7726        let run = self.state.id.clone();
7727        for (i, spec) in chain.into_iter().enumerate() {
7728            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7729            let github_spec = match agent::for_github_text(
7730                &spec,
7731                &self.state.config.graph.github_language,
7732                &winner.worktree,
7733            ) {
7734                Ok(spec) => spec,
7735                Err(e) => {
7736                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7737                    continue;
7738                }
7739            };
7740            let job = SeatJob {
7741                spec: github_spec,
7742                seat,
7743                cwd: winner.worktree.clone(),
7744                prompt: prompt.clone(),
7745                timeout,
7746                allow_write: false,
7747                sessions,
7748                artifacts: agent::artifacts_dir(&self.state.dir()),
7749                stem: if i == 0 {
7750                    "change-summary".to_owned()
7751                } else {
7752                    format!("change-summary-{}", spec.id)
7753                },
7754                handover: None,
7755            };
7756            let ctx = WaveCtx {
7757                run: &run,
7758                node: "pr-summary",
7759                prompts: &prompts,
7760                cache: cache.as_deref(),
7761                round: None,
7762                carry_seats: false,
7763                pauses: Some((&self.pause, &self.interrupt)),
7764            };
7765            let (seat, outcome) =
7766                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7767            self.state.seats.insert(seat.key.clone(), seat);
7768            let text = match outcome {
7769                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7770                AgentOutcome::ResourceBlocked(why) => {
7771                    // No agent was asked: the run stops resumably before
7772                    // anything is published (see `guarded_pr_message`).
7773                    if self.state.resource_block.is_none() {
7774                        self.state.resource_block = Some(crate::run::ResourceBlock {
7775                            cause: crate::run::ResourceBlockCause::Other,
7776                            node: "pr-summary".to_owned(),
7777                            kind: "shared resource".to_owned(),
7778                            detail: why,
7779                            at: Timestamp::now(),
7780                        });
7781                    }
7782                    return Err("a shared resource was busy".to_owned());
7783                }
7784                _ => continue,
7785            };
7786            let blind = &self.state.config.blind;
7787            let Some(title) = summary_title(&text) else {
7788                continue;
7789            };
7790            let body: String = summary_without_title(&text)
7791                .lines()
7792                .filter(|l| !l.contains("magi:run/"))
7793                .collect::<Vec<_>>()
7794                .join("\n");
7795            let body = blind::sanitize_prose(body.trim(), blind);
7796            let title = blind::sanitize_prose(&title, blind);
7797            if body.trim().is_empty() || title.trim().is_empty() {
7798                continue;
7799            }
7800            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7801                continue;
7802            }
7803            // The branch must still be the one the agent was shown.
7804            if git::rev_parse(&winner.worktree, "HEAD")
7805                .await
7806                .ok()
7807                .as_deref()
7808                != Some(head)
7809            {
7810                return Err("the branch moved while the summary was written".to_owned());
7811            }
7812            if git::merge_base(&winner.worktree, &tracking, head)
7813                .await
7814                .ok()
7815                .as_deref()
7816                != Some(base.as_str())
7817                || crate::pr_outcome::summary_evidence(
7818                    &self.state,
7819                    crate::pr_outcome::FinalHead::At(head),
7820                ) != evidence
7821            {
7822                return Err("the summary evidence changed while it was written".to_owned());
7823            }
7824            self.state.change_summary = Some(crate::run::ChangeSummary {
7825                head: head.to_owned(),
7826                base,
7827                evidence: fingerprint,
7828                title,
7829                body,
7830            });
7831            self.state.event(
7832                "pr-summary",
7833                format!("change summary written by {} for the final diff", spec.id),
7834            );
7835            if let Err(e) = self.state.save() {
7836                tracing::warn!("could not save the change summary: {e:#}");
7837            }
7838            return Ok(());
7839        }
7840        self.state
7841            .event("pr-summary", "no agent produced a change summary");
7842        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7843    }
7844
7845    /// One read-only rewrite request for a withheld description. The
7846    /// implementer's own conversation is tried first when it exists and can be
7847    /// resumed; then the `[roles] language_judge` chain (or the default pick)
7848    /// writes it in a separate conversation, skipping the implementer's agent.
7849    /// The prompt carries the whole text, so no agent needs any memory. Each
7850    /// agent is asked at most once and the chain advances on an error, quota,
7851    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
7852    /// that has run out of quota must not end the rewrite. Nothing is recorded
7853    /// as a handover.
7854    async fn rewrite_pr_text(
7855        &mut self,
7856        winner: &Candidate,
7857        pr: &PrMessage,
7858        violations: &[crate::github_text::Violation],
7859    ) -> Option<String> {
7860        let prompt = format!(
7861            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7862            pr.title, pr.body
7863        );
7864        let sessions = self.state.config.graph.sessions;
7865        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7866        // (spec, the resumed implementer's seat). A fallback seat is created
7867        // only when its turn comes, so an unused fallback leaves no record.
7868        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7869        let key = format!("impl-{}", winner.label);
7870        if let Some(spec) = self
7871            .state
7872            .config
7873            .agents
7874            .iter()
7875            .find(|a| a.id == winner.agent)
7876            .cloned()
7877            && let Some(existing) = self.state.seats.get(&key)
7878            && existing.agent == spec.id
7879            && agent::has_session(spec.kind, existing, sessions)
7880        {
7881            attempts.push((spec, Some(existing.clone())));
7882        }
7883        match agent::pick_chain(
7884            &self.state.config.agents,
7885            self.state.config.roles.language_judge.as_ref(),
7886            &agent::installed,
7887            "language_judge",
7888        ) {
7889            Ok(chain) => {
7890                for spec in chain {
7891                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7892                        continue;
7893                    }
7894                    attempts.push((spec, None));
7895                }
7896            }
7897            Err(e) if attempts.is_empty() => {
7898                tracing::warn!("no agent for the description rewrite: {e:#}");
7899                self.state
7900                    .event("github-text", "no rewrite agent available");
7901                return None;
7902            }
7903            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7904        }
7905        let prompts = self.state.config.prompts.clone();
7906        let cache = self.state.config.cache_dir();
7907        let run = self.state.id.clone();
7908        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7909            let resumed = resumed_seat.is_some();
7910            let seat = resumed_seat.unwrap_or_else(|| {
7911                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7912            });
7913            let github_spec = match agent::for_github_text(
7914                &spec,
7915                &self.state.config.graph.github_language,
7916                &winner.worktree,
7917            ) {
7918                Ok(spec) => spec,
7919                Err(e) => {
7920                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7921                    continue;
7922                }
7923            };
7924            let job = SeatJob {
7925                spec: github_spec,
7926                seat,
7927                cwd: winner.worktree.clone(),
7928                prompt: prompt.clone(),
7929                timeout: retry_budget(timeout_full, resumed),
7930                allow_write: false,
7931                sessions,
7932                artifacts: agent::artifacts_dir(&self.state.dir()),
7933                stem: if i == 0 {
7934                    "github-text-rewrite".to_owned()
7935                } else {
7936                    format!("github-text-rewrite-{}", spec.id)
7937                },
7938                handover: None,
7939            };
7940            let ctx = WaveCtx {
7941                run: &run,
7942                node: "github-text",
7943                prompts: &prompts,
7944                cache: cache.as_deref(),
7945                round: None,
7946                carry_seats: false,
7947                pauses: Some((&self.pause, &self.interrupt)),
7948            };
7949            let (seat, outcome) =
7950                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7951            self.state.seats.insert(seat.key.clone(), seat);
7952            match outcome {
7953                AgentOutcome::Ok(o)
7954                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7955                {
7956                    return Some(o.text);
7957                }
7958                AgentOutcome::ResourceBlocked(_) => return None,
7959                _ => {}
7960            }
7961        }
7962        None
7963    }
7964
7965    /// Abandon the standing posting-gate question, if any.
7966    fn retire_github_text_question(&mut self, why: &str) {
7967        let Some(id) = self
7968            .state
7969            .github_text
7970            .as_ref()
7971            .and_then(|g| g.question.clone())
7972        else {
7973            return;
7974        };
7975        let _ = ask::Questions::open().update(&id, |q| {
7976            q.abandon(why);
7977            Ok(())
7978        });
7979    }
7980
7981    /// Carry a withheld text to its end: file the question, park while it is
7982    /// open, and on an answer, silence or timeout pick the text to post. The
7983    /// decision is saved before it is used, so it applies once.
7984    async fn settle_withheld(
7985        &mut self,
7986        winner: &Candidate,
7987        pr: &PrMessage,
7988        mut g: crate::run::GithubTextAsk,
7989    ) -> Result<Option<PrMessage>> {
7990        use crate::github_text as gt;
7991        let store = ask::Questions::open();
7992        let timeout = self.state.config.graph.answer_timeout;
7993        if !g.resolved {
7994            let mut standing = g
7995                .question
7996                .as_ref()
7997                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7998            if let Some(q) = standing.as_ref()
7999                && q.status.open()
8000                && gt::expired(q, timeout)
8001            {
8002                let why = format!("no answer within {}s of asking", timeout.max(1));
8003                standing = store
8004                    .update(&q.id, |q| {
8005                        q.abandon(&why);
8006                        Ok(())
8007                    })
8008                    .ok()
8009                    .map(|(q, ())| q);
8010            }
8011            let mut ask_again: Option<Option<Vec<String>>> = None;
8012            match standing {
8013                // Recorded but never written (a stop between the two saves),
8014                // or removed: ask again without spending the retry.
8015                None if g.asks <= 1 => {
8016                    g.asks = 0;
8017                    ask_again = Some(None);
8018                }
8019                None => {
8020                    g.resolved = true;
8021                    g.chosen_title = None;
8022                    g.choice = Some(GithubTextChoice::Fallback);
8023                }
8024                Some(q) if q.status.open() => {
8025                    self.park_on_github_text(&g)?;
8026                    return Ok(None);
8027                }
8028                Some(q) => match gt::read_reply(&q) {
8029                    gt::Reply::Fallback => {
8030                        g.resolved = true;
8031                        g.chosen_title = None;
8032                        g.choice = Some(GithubTextChoice::Fallback);
8033                    }
8034                    gt::Reply::Original => {
8035                        g.resolved = true;
8036                        g.chosen_title = None;
8037                        g.choice = Some(GithubTextChoice::Original);
8038                    }
8039                    gt::Reply::Rewrite => {
8040                        g.resolved = true;
8041                        g.chosen_title = None;
8042                        g.choice = Some(if g.rewrite.is_some() {
8043                            GithubTextChoice::Rewrite
8044                        } else {
8045                            GithubTextChoice::Fallback
8046                        });
8047                    }
8048                    gt::Reply::Title(text) => {
8049                        let first = text
8050                            .lines()
8051                            .map(str::trim)
8052                            .find(|l| !l.is_empty())
8053                            .unwrap_or("");
8054                        let decision = if first.is_empty() {
8055                            None
8056                        } else {
8057                            let candidate = PrMessage {
8058                                title: first.to_owned(),
8059                                body: String::new(),
8060                            };
8061                            self.judge_pr_language(&winner.worktree, &candidate).await
8062                        };
8063                        match gt::vet_title(
8064                            &text,
8065                            decision,
8066                            self.state.config.graph.github_english(),
8067                        ) {
8068                            Ok(title) => {
8069                                g.resolved = true;
8070                                g.chosen_title = Some(title);
8071                                g.choice = Some(GithubTextChoice::MyTitle);
8072                            }
8073                            Err(cats) if g.asks < 2 => {
8074                                ask_again = Some(Some(
8075                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8076                                ))
8077                            }
8078                            Err(_) => {
8079                                g.resolved = true;
8080                                g.chosen_title = None;
8081                                g.choice = Some(GithubTextChoice::Fallback);
8082                            }
8083                        }
8084                    }
8085                },
8086            }
8087            if let Some(rejected) = ask_again {
8088                // A retry (after a rejected replacement title) is still about
8089                // the original text: the same ORIGINAL / FALLBACK comparison.
8090                let w = gt::Withheld {
8091                    title: g.title,
8092                    body: g.body,
8093                    categories: g.categories.clone(),
8094                };
8095                // Named by the original text's fingerprint alone, so a resume
8096                // rewrites the same file and the detail stays identical.
8097                let artifact = match crate::run::write_artifact(
8098                    &self.state,
8099                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8100                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8101                ) {
8102                    Ok(p) => Some(p),
8103                    Err(e) => {
8104                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8105                        None
8106                    }
8107                };
8108                let (detail, comparison) = gt::question_content(
8109                    &w,
8110                    &pr.title,
8111                    &pr.body,
8112                    &self.state.id,
8113                    rejected.as_deref(),
8114                    artifact.as_deref(),
8115                    g.rewrite.as_ref(),
8116                );
8117                let mut q = ask::Question::new(
8118                    self.state.id.clone(),
8119                    gt::ASK_NODE.to_owned(),
8120                    gt::ASK_SEAT.to_owned(),
8121                    gt::question_summary(&self.state.config.graph.language, &w),
8122                    detail,
8123                    gt::question_choices(&w, g.rewrite.as_ref()),
8124                );
8125                q.posting_comparison = Some(comparison);
8126                q.answer_timeout = timeout;
8127                // Recorded before it is written: a stop in between leaves a
8128                // record naming a question the resume then files again.
8129                g.question = Some(q.id.clone());
8130                g.asks += 1;
8131                self.state.github_text = Some(g.clone());
8132                self.state.event(
8133                    "github-text",
8134                    format!("asking the owner about the withheld text ({})", q.short()),
8135                );
8136                self.state.save()?;
8137                store
8138                    .put_executable(&mut q)
8139                    .context("file the posting-gate question")?;
8140                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8141                    tracing::warn!(
8142                        "could not notify about posting-gate question {}: {e:#}",
8143                        q.short()
8144                    );
8145                }
8146                self.park_on_github_text(&g)?;
8147                return Ok(None);
8148            }
8149        }
8150        let resolved = g.chosen_title.clone();
8151        let original = g.choice == Some(GithubTextChoice::Original);
8152        // Field names only: the event log never records the text.
8153        self.state.event(
8154            "github-text",
8155            if original {
8156                format!(
8157                    "posting the original text for: {}",
8158                    match (g.title, g.body) {
8159                        (true, true) => "title, description",
8160                        (true, false) => "title",
8161                        _ => "description",
8162                    }
8163                )
8164            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8165                "posting the saved rewrite title and description".to_owned()
8166            } else if resolved.is_some() {
8167                "posting the owner's replacement title".to_owned()
8168            } else {
8169                "posting the neutral text for the withheld fields".to_owned()
8170            },
8171        );
8172        self.state.github_text = Some(g.clone());
8173        self.state.save()?;
8174        if g.choice == Some(GithubTextChoice::Rewrite)
8175            && let Some(r) = g.rewrite.as_ref()
8176        {
8177            return Ok(Some(PrMessage {
8178                title: r.title.clone(),
8179                body: r.body.clone(),
8180            }));
8181        }
8182        let w = gt::Withheld {
8183            title: g.title,
8184            body: g.body,
8185            categories: g.categories.clone(),
8186        };
8187        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8188        if let Some(t) = resolved {
8189            title = t;
8190        }
8191        // Every withheld field is now neutral or vetted English.
8192        let english = gt::LanguageDecision {
8193            title_english: true,
8194            body_english: true,
8195            body_complete: true,
8196        };
8197        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8198        if original {
8199            // The owner's explicit choice overrides the gate for the fields it
8200            // was made for; the others keep what the gate produced.
8201            if g.title {
8202                title = pr.title.clone();
8203            }
8204            if g.body {
8205                body = pr.body.clone();
8206            }
8207        }
8208        Ok(Some(PrMessage { title, body }))
8209    }
8210
8211    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8212        self.state.github_text = Some(g.clone());
8213        self.state.end_pass_parked(ParkCause::Question);
8214        self.state.parked = true;
8215        self.state.event(
8216            "github-text",
8217            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8218        );
8219        self.state.save()
8220    }
8221
8222    /// Ask `[roles] language_judge` about this text and record which source
8223    /// decided; the text itself is never recorded.
8224    async fn judge_pr_language(
8225        &mut self,
8226        cwd: &Path,
8227        pr: &PrMessage,
8228    ) -> Option<crate::github_text::LanguageDecision> {
8229        self.state.config.roles.language_judge.as_ref()?;
8230        if !self.state.config.graph.language_gate() {
8231            return None;
8232        }
8233        let decision =
8234            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8235        self.state.event(
8236            "github-text",
8237            if decision.is_some() {
8238                "language decided by the language judge"
8239            } else {
8240                "language judge unavailable; using built-in heuristics"
8241            },
8242        );
8243        decision
8244    }
8245
8246    async fn merge(&mut self) -> Result<()> {
8247        // Same reasoning as `gate`: ask the review and gate records directly
8248        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8249        // skip can rewrite on reentry to something that no longer says
8250        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8251        // so a hand-off (open findings, green verification) reaches merge
8252        // exactly like a genuinely clean round does.
8253        //
8254        // A run resumed mid-`land` never reaches here at all: `execute`
8255        // recognises `RunStatus::Landing` before it even calls `prep`, and
8256        // routes straight to `run_land` instead. That has to happen a level
8257        // up from this function, not with a check in here, because
8258        // `review_loop`'s own status recomputation (see its doc) runs
8259        // *before* `merge` on every reentry and would otherwise overwrite
8260        // the `Landing` marker with `Gating` before this node ever saw it.
8261        if self
8262            .state
8263            .base_sync
8264            .as_ref()
8265            .is_some_and(|s| s.conflict.is_some())
8266            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8267                != Some(RunStatus::Gating)
8268            // `gate_ran == false` is not "passed" - `gate` leaves it false
8269            // both before it has ever run and when its last attempt was
8270            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8271            // permission to merge on nothing but the review record. Only a
8272            // gate that actually ran may proceed; an intentional local
8273            // review can also finish with verification explicitly not configured.
8274            // `RunState::gate_status` computes that distinction.
8275            || !self.state.gate_allows_progress()
8276        {
8277            return Ok(());
8278        }
8279        // This node's own record, not `status`: `status == Ready` is not
8280        // unique to the harmless `MergeMode::None` path this line was
8281        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8282        // run's PR was closed without merging — and on that run `mode` is
8283        // still `Pr`, so a reentry that fell through here would push and
8284        // open a second pull request. `self.state.merge` is set exactly once
8285        // this node (or `land`) has already produced a verdict, under every
8286        // mode, which is what "already done" actually means here.
8287        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8288            return Ok(());
8289        }
8290        let Some(winner) = self.state.winner().cloned() else {
8291            return Ok(());
8292        };
8293        let repo = self.state.repo.clone();
8294        let base = self.state.base_branch.clone();
8295        let mode = self.state.config.merge.mode;
8296        let style = self.state.config.merge.style;
8297        let facts = if is_review_run(&self.state) {
8298            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8299            let start = review_base(
8300                &repo,
8301                &self.state.config.merge.remote,
8302                &base,
8303                &self.state.base_commit,
8304                &winner.branch,
8305            )
8306            .await;
8307            branch_facts(&repo, &start, &winner.branch).await
8308        } else {
8309            None
8310        };
8311        // `None` when the base could not be freshly read: then an adopted
8312        // pull request's title is left alone.
8313        let leaked = if is_review_run(&self.state) {
8314            leaked_subjects(&self.state, &winner.branch).await
8315        } else {
8316            Some(Vec::new())
8317        };
8318        // `None`: parked on the owner's word about a withheld title or body.
8319        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8320        // the resume arrives here again.
8321        let Some(pr) = self
8322            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8323            .await?
8324        else {
8325            return Ok(());
8326        };
8327        let message = pr.commit_message();
8328
8329        let outcome = match mode {
8330            MergeMode::None => MergeOutcome {
8331                mode,
8332                ok: true,
8333                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8334                empty: false,
8335            },
8336            MergeMode::Pr | MergeMode::Local
8337                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8338            {
8339                MergeOutcome {
8340                    mode,
8341                    ok: false,
8342                    detail: empty_candidate_detail(&self.state, &base),
8343                    empty: true,
8344                }
8345            }
8346            MergeMode::Local => {
8347                let on = git::current_branch(&repo).await?;
8348                if on.as_deref() != Some(base.as_str()) {
8349                    MergeOutcome {
8350                        mode,
8351                        ok: false,
8352                        detail: format!(
8353                            "{} has {} checked out, not the base branch {base}",
8354                            repo.display(),
8355                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8356                        ),
8357                        empty: false,
8358                    }
8359                } else if !git::is_clean(&repo).await? {
8360                    MergeOutcome {
8361                        mode,
8362                        ok: false,
8363                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8364                        empty: false,
8365                    }
8366                } else {
8367                    let out = match style {
8368                        MergeStyle::Merge => {
8369                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8370                        }
8371                        MergeStyle::Squash => {
8372                            git::merge_squash(&repo, &winner.branch, &message).await?
8373                        }
8374                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8375                    };
8376                    MergeOutcome {
8377                        mode,
8378                        ok: out.ok(),
8379                        detail: if out.ok() { out.stdout } else { out.stderr },
8380                        empty: false,
8381                    }
8382                }
8383            }
8384            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8385                self.state.pr_publication_pending = true;
8386                self.state.save()?;
8387                let out =
8388                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8389                self.state.pr_publication_pending = out.is_err();
8390                MergeOutcome {
8391                    mode,
8392                    ok: out.is_ok(),
8393                    detail: match out {
8394                        Ok(url) => url,
8395                        Err(e) => format!("{e:#}"),
8396                    },
8397                    empty: false,
8398                }
8399            }
8400            MergeMode::Pr => {
8401                let remote = self.state.config.merge.remote.clone();
8402                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8403                if !pushed.ok() {
8404                    MergeOutcome {
8405                        mode,
8406                        ok: false,
8407                        detail: pushed.stderr,
8408                        empty: false,
8409                    }
8410                } else {
8411                    // A retry or resume of a run whose branch already has an
8412                    // open pull request adopts it rather than failing on a
8413                    // duplicate. Only this winner branch into this base:
8414                    // `branch_for` derives the name from the run id, so a
8415                    // different run's pull request never matches.
8416                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8417                    let out = match pr_merge_plan(found) {
8418                        PrPlan::Create => {
8419                            gh_pr_create(
8420                                &winner.worktree,
8421                                &base,
8422                                &winner.branch,
8423                                &pr.title,
8424                                &pr.body,
8425                            )
8426                            .await
8427                        }
8428                        PrPlan::Adopt { url, title } => {
8429                            self.state
8430                                .event("merge", format!("Pr: adopted open pull request {url}"));
8431                            if self.state.github_text.as_ref().is_some_and(|g| {
8432                                g.resolved
8433                                    && g.choice == Some(GithubTextChoice::Rewrite)
8434                                    && g.rewrite.is_some()
8435                            }) {
8436                                // The owner chose the entire pair, even if the
8437                                // existing title already matches or was edited.
8438                                land::set_pr_text_verbatim(
8439                                    &winner.worktree,
8440                                    &url,
8441                                    &pr.title,
8442                                    &pr.body,
8443                                )
8444                                .await
8445                                .map(|()| url)
8446                            } else {
8447                                if title != pr.title
8448                                    && (!is_review_run(&self.state)
8449                                        || leaked
8450                                            .as_deref()
8451                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8452                                    && let Err(e) =
8453                                        if self.state.github_text.as_ref().is_some_and(|g| {
8454                                            g.resolved
8455                                                && g.title
8456                                                && g.choice == Some(GithubTextChoice::Original)
8457                                        }) {
8458                                            // The owner chose to post this title unmodified.
8459                                            land::set_pr_title_verbatim(
8460                                                &winner.worktree,
8461                                                &url,
8462                                                &pr.title,
8463                                            )
8464                                            .await
8465                                        } else {
8466                                            land::set_pr_title(
8467                                                &mut self.state,
8468                                                &winner.worktree,
8469                                                &url,
8470                                                &pr.title,
8471                                            )
8472                                            .await
8473                                        }
8474                                {
8475                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8476                                    self.state
8477                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8478                                }
8479                                Ok(url)
8480                            }
8481                        }
8482                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8483                    };
8484                    match out {
8485                        Ok(url) => MergeOutcome {
8486                            mode,
8487                            ok: true,
8488                            detail: url,
8489                            empty: false,
8490                        },
8491                        Err(e) => MergeOutcome {
8492                            mode,
8493                            ok: false,
8494                            detail: e.to_string(),
8495                            empty: false,
8496                        },
8497                    }
8498                }
8499            }
8500        };
8501
8502        self.state.status = match (mode, outcome.ok) {
8503            (MergeMode::None, _) => RunStatus::Ready,
8504            (_, true) if outcome.empty => RunStatus::Ready,
8505            (MergeMode::Pr, true) => RunStatus::Ready,
8506            (_, true) => RunStatus::Merged,
8507            (_, false) => RunStatus::Blocked,
8508        };
8509        self.state.event(
8510            "merge",
8511            format!(
8512                "{:?}: {}",
8513                mode,
8514                outcome.detail.lines().next().unwrap_or("")
8515            ),
8516        );
8517        self.state.merge = Some(outcome);
8518        self.state.save()?;
8519
8520        // The PR is open and the run would historically stop here, leaving the
8521        // operator to watch checks, feed review comments back to a fixer, and
8522        // merge. That was done by hand six times in one session before this
8523        // existed. Opt-in, because merging is the one irreversible thing magi
8524        // can do to a repository.
8525        if self.state.config.graph.land
8526            && mode == MergeMode::Pr
8527            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8528        {
8529            self.run_land().await?;
8530        }
8531        // `run_land` may have left `status` at `Landing` - still waiting on
8532        // CI or the owner's approval, not actually settled - so this has to
8533        // read whatever `status` ended up as here, not the `Merged` this
8534        // function set a few lines up.
8535        self.settle_questions();
8536        Ok(())
8537    }
8538
8539    /// Enter `land`.
8540    ///
8541    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8542    /// a resumed run's re-entry. `land::land` itself is what serialises the
8543    /// two git-mutating moments inside the loop — the rebase push and
8544    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8545    /// to hold a lock across the whole call, and doing so would serialise
8546    /// this run's CI wait against a *different* run's land-approval resume
8547    /// in the same repository, which is exactly the "must not wait on
8548    /// another task" property the daemon's slot-freeing exists to give.
8549    async fn run_land(&mut self) -> Result<()> {
8550        crate::pr_head::ensure_owner(&self.state)?;
8551        let url = self
8552            .state
8553            .pr
8554            .as_ref()
8555            .map(|pr| pr.url.clone())
8556            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8557            .unwrap_or_default();
8558        let url = url.lines().next().unwrap_or("").trim().to_owned();
8559        if !url.starts_with("http") {
8560            return Ok(());
8561        }
8562        // A land failure is not a lost run: the work is on a branch and the
8563        // pull request is open, which is exactly where a human takes over.
8564        loop {
8565            match land::land(&mut self.state, &url).await {
8566                Ok(_)
8567                    if self
8568                        .state
8569                        .merge_repairs
8570                        .last()
8571                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8572                {
8573                    if self.repair_merge_review().await? {
8574                        continue;
8575                    }
8576                }
8577                Ok(pr) if self.state.parked => {
8578                    // `land` already saved the parked marker; nothing here
8579                    // overrides `status` back to a terminal value while an
8580                    // approval is still outstanding.
8581                    let _ = pr;
8582                }
8583                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8584                    self.state.status = RunStatus::Superseded;
8585                    self.state.save()?;
8586                }
8587                Ok(pr) => {
8588                    self.state.status = match pr.state {
8589                        land::PrLifecycle::Merged => RunStatus::Merged,
8590                        _ => RunStatus::Blocked,
8591                    };
8592                    // Downstream of a confirmed merge only - see
8593                    // `bump::should_release_bump`'s own doc for why this one
8594                    // check covers all three of `land`'s success paths.
8595                    // Best-effort: the run already landed, so a failure here
8596                    // (the decision call, `gh`, `cargo`) is recorded and never
8597                    // turns a landed run into a failed one.
8598                    if bump::should_release_bump(self.state.status)
8599                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8600                    {
8601                        // The event is the run's own record. Not-eligible cases
8602                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8603                        // `Err` is a bump that was tried and failed:
8604                        // `after_merge` itself raises the operator notice for
8605                        // that, whether or not a release PR exists yet.
8606                        self.state
8607                            .event("bump", format!("release bump skipped: {e:#}"));
8608                    }
8609                    // Independent of the bump, and best-effort in the same way:
8610                    // findings the merge left open become follow-up tasks.
8611                    if self.state.status == RunStatus::Merged {
8612                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8613                    }
8614                    self.state.save()?;
8615                }
8616                Err(e) => {
8617                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8618                        return Err(e);
8619                    }
8620                    self.state.status = RunStatus::Blocked;
8621                    self.state.event("land", format!("gave up: {e}"));
8622                    self.state.save()?;
8623                }
8624            }
8625            break;
8626        }
8627        Ok(())
8628    }
8629
8630    /// One repair cycle stays under the existing driver/task claim. An
8631    /// interrupted fixer is rolled back and held rather than run twice.
8632    async fn repair_merge_review(&mut self) -> Result<bool> {
8633        use crate::run::MergeRepairPhase as Phase;
8634        crate::pr_head::ensure_owner(&self.state)?;
8635        let i = self.state.merge_repairs.len() - 1;
8636        let request = self.state.merge_repairs[i].clone();
8637        if request.phase == Phase::Fixing {
8638            let restored = match self.state.winner().cloned() {
8639                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8640                None => Err(anyhow::anyhow!("repair has no winning branch")),
8641            };
8642            let why = match restored {
8643                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8644                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8645                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8646            };
8647            self.hold_merge_repair(i, why)?;
8648            return Ok(false);
8649        }
8650        let failure = if i >= self.state.config.graph.land_rounds
8651            || self.state.config.graph.review_rounds == 0
8652        {
8653            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8654        } else {
8655            None
8656        };
8657        if let Some(why) = failure {
8658            self.hold_merge_repair(i, why)?;
8659            return Ok(false);
8660        }
8661        if request.phase == Phase::Requested {
8662            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8663                Ok(pr) => pr,
8664                Err(e) => {
8665                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8666                    return Ok(false);
8667                }
8668            };
8669            self.state.merge_repairs[i].phase = Phase::Fixing;
8670            self.state.land_approval = None;
8671            self.state.land_armed_head = None;
8672            self.state.save()?;
8673            // Serialize the exact records, including finding ids/details,
8674            // reconsideration reasons, final votes and verification output.
8675            let evidence = serde_json::to_string_pretty(&request)?;
8676            let reason = format!(
8677                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8678            );
8679            let budget = self.state.config.graph.land_rounds;
8680            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8681            match result {
8682                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8683                Ok(land::Fixed::Failed(why)) => {
8684                    self.hold_merge_repair(
8685                        i,
8686                        format!(
8687                            "repair failed: {why}; inspect the branch and run an explicit review"
8688                        ),
8689                    )?;
8690                    return Ok(false);
8691                }
8692                Err(e) => {
8693                    self.hold_merge_repair(
8694                        i,
8695                        format!(
8696                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8697                        ),
8698                    )?;
8699                    return Ok(false);
8700                }
8701            }
8702            // Even no change gets a real new reviewer wave. The previous
8703            // evidence remains in the request; a former clean/concluded round
8704            // cannot short-circuit the new review or gate.
8705            self.begin_merge_repair_review(i)?;
8706        }
8707        self.review_loop().await?;
8708        self.gate().await?;
8709        if self.resource_halt()? || self.state.parked {
8710            return Ok(false);
8711        }
8712        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8713            != Some(RunStatus::Gating)
8714            || !self.state.gate_allows_progress()
8715        {
8716            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8717            return Ok(false);
8718        }
8719        // Review and gate may have added commits after the initial repair.
8720        // Publish exactly their final head to the existing PR, with a lease.
8721        let winner = self
8722            .state
8723            .winner()
8724            .cloned()
8725            .context("repair has no winning branch")?;
8726        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8727        if self.state.gate_head.as_deref() != Some(after.as_str())
8728            || !git::is_clean(&winner.worktree).await?
8729        {
8730            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8731            return Ok(false);
8732        }
8733        let expected = self
8734            .state
8735            .pr_pushed_head
8736            .as_deref()
8737            .unwrap_or(&request.head)
8738            .to_owned();
8739        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8740        // Push the checked commit itself: a local ref moving after the check
8741        // must not change which commit the forge receives.
8742        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8743        let refspec = format!("{after}:refs/heads/{public}");
8744        let push = git::git_raw(
8745            &winner.worktree,
8746            &["push", &lease, &self.state.config.merge.remote, &refspec],
8747        )
8748        .await;
8749        match push {
8750            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8751            Ok(out) => {
8752                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8753                return Ok(false);
8754            }
8755            Err(e) => {
8756                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8757                return Ok(false);
8758            }
8759        }
8760        self.state.merge_repairs[i].phase = Phase::Complete;
8761        self.state.status = RunStatus::Landing;
8762        self.state.event(
8763            "repair",
8764            "new review and gate completed; fresh owner merge decision required",
8765        );
8766        self.state.save()?;
8767        Ok(true)
8768    }
8769
8770    fn merge_repair_artifacts(&self) -> PathBuf {
8771        let artifacts = agent::artifacts_dir(&self.state.dir());
8772        match self
8773            .state
8774            .merge_repairs
8775            .last()
8776            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8777        {
8778            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8779            None => artifacts,
8780        }
8781    }
8782
8783    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8784        self.state.reviews.clear();
8785        self.state.gate.clear();
8786        self.state.gate_ran = false;
8787        self.state.gate_head = None;
8788        self.state.gate_fixes.clear();
8789        self.state.gate_repair_e2e = None;
8790        self.state.pre_gate.clear();
8791        self.state.pre_gate_commit = None;
8792        self.state.contested_handoff = None;
8793        self.state.land_reruns = None;
8794        self.state.land_approval = None;
8795        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8796        self.state.status = RunStatus::Reviewing;
8797        self.state.save()
8798    }
8799
8800    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8801        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8802        self.state.merge_repairs[index].failure = Some(why.clone());
8803        self.state.status = RunStatus::Blocked;
8804        self.state.event("repair", why);
8805        self.state.save()
8806    }
8807
8808    // -------------------------------------------------------------- helpers
8809
8810    /// Fetch or create a seat, keeping its conversation across nodes.
8811    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8812        if let Some(existing) = self.state.seats.get(key)
8813            && existing.agent == agent
8814        {
8815            return existing.clone();
8816        }
8817        // A seat that changes agent mints its session id from the agent too,
8818        // like a handover: the old agent's uuid is already taken by the CLI.
8819        let fresh = if self.state.seats.contains_key(key) {
8820            handover_seat(key, agent, self.state.next_seat_seed())
8821        } else {
8822            SeatState::new(key, agent, self.state.seed)
8823        };
8824        self.state.seats.insert(key.to_owned(), fresh.clone());
8825        fresh
8826    }
8827
8828    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8829    /// seat to another roster agent, in which case that agent. Nodes that
8830    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8831    /// reconsideration) must keep talking to whoever answered it, not slip
8832    /// back to the agent that failed it.
8833    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8834        match self.state.seats.get(key) {
8835            Some(s) if s.agent != spec.id => {
8836                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8837            }
8838            _ => spec,
8839        }
8840    }
8841
8842    /// A candidate rendered for judging, with the leak policy applied.
8843    fn view(&self, c: &Candidate) -> CandidateView {
8844        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8845            .unwrap_or_default();
8846        let (patch, _) = blind::sanitize_patch(
8847            &format!("candidate {} patch", c.label),
8848            &raw,
8849            &self.state.config.blind,
8850        );
8851        CandidateView {
8852            label: c.label,
8853            branch: c.branch.clone(),
8854            summary: c.summary.clone(),
8855            stat: c.stat.clone(),
8856            patch,
8857        }
8858    }
8859
8860    /// The full candidate set as prompt text, for seats with no live session.
8861    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8862        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8863        prompt::judge(
8864            "(see above)",
8865            &views,
8866            self.roles.judges.len(),
8867            base_short,
8868            "en",
8869        )
8870    }
8871
8872    /// The final-vote prompt with everything a seat that has no session of its
8873    /// own needs: the candidates, the seat's own ranking and reasons, and the
8874    /// anonymised deliberation it took part in (only when there was one, so a
8875    /// handed-over seat never sees more than the seat it replaces did). The
8876    /// `Final vote` heading stays first.
8877    fn vote_prompt_full(
8878        &self,
8879        j: usize,
8880        labels: &[char],
8881        language: &str,
8882        candidates: &[Candidate],
8883        base_short: &str,
8884    ) -> String {
8885        let mut text = format!(
8886            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8887            prompt::final_vote(labels, language),
8888            self.state.instruction,
8889            self.candidate_block(candidates, base_short)
8890        );
8891        if let Some(own) = self
8892            .state
8893            .judgements
8894            .get(j)
8895            .filter(|r| !r.ranking.is_empty())
8896        {
8897            let reasons = own
8898                .reasons
8899                .iter()
8900                .map(|(k, v)| format!("- {k}: {v}"))
8901                .collect::<Vec<_>>()
8902                .join("\n");
8903            text.push_str(&format!(
8904                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8905                own.ranking.iter().collect::<String>(),
8906                if reasons.is_empty() {
8907                    ""
8908                } else {
8909                    ", because:\n"
8910                }
8911            ));
8912        }
8913        if !self.state.deliberation.is_empty() {
8914            text.push_str("\n# What was argued before this vote\n");
8915            for t in self.transcript(&[], j) {
8916                text.push_str(&format!(
8917                    "\n## {}{}\n\n{}\n",
8918                    t.who,
8919                    if t.is_self { " (you)" } else { "" },
8920                    t.body.trim()
8921                ));
8922            }
8923        }
8924        text
8925    }
8926
8927    /// Anonymised transcript for judge `self_idx`.
8928    ///
8929    /// The initial rankings are always the opening statements. Seeding them
8930    /// only when no turn had been taken yet meant every judge after the first
8931    /// argued against a single voice instead of against the actual split — the
8932    /// disagreement is the information, so it is always on the table.
8933    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8934        let mut turns = Vec::new();
8935        for j in &self.state.judgements {
8936            if j.ranking.is_empty() {
8937                continue;
8938            }
8939            let reasons = j
8940                .reasons
8941                .iter()
8942                .map(|(k, v)| format!("- {k}: {v}"))
8943                .collect::<Vec<_>>()
8944                .join("\n");
8945            turns.push(Turn {
8946                who: format!("Judge {} (opening ranking)", j.judge),
8947                is_self: j.judge == self_idx + 1,
8948                body: format!(
8949                    "Ranked {}{}{reasons}",
8950                    j.ranking.iter().collect::<String>(),
8951                    if reasons.is_empty() {
8952                        ""
8953                    } else {
8954                        ", because:\n"
8955                    }
8956                ),
8957            });
8958        }
8959        for t in self
8960            .state
8961            .deliberation
8962            .iter()
8963            .flat_map(|r| r.turns.iter())
8964            .chain(current)
8965        {
8966            turns.push(Turn {
8967                who: format!("Judge {}", t.judge),
8968                is_self: t.judge == self_idx + 1,
8969                body: t.body.clone(),
8970            });
8971        }
8972        turns
8973    }
8974}
8975
8976/// Does this seat still hold the context a follow-up prompt would rely on?
8977fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8978    agent::has_session(spec.kind, seat, sessions)
8979}
8980
8981/// The next entry in `roster` after `start`, never wrapping back to the
8982/// front, whose id is not in `tried` yet.
8983///
8984/// Starts one past `start` rather than at the front of `roster`: `start` is
8985/// the seat's own original position, and a seat whose candidate slot already
8986/// sits on the roster's second entry must fall through to the third next, not
8987/// restart at the first — which is very likely a different candidate's own
8988/// agent already. Never wraps back past `start`, for the same reason: an
8989/// entry earlier in the roster than the seat's own position is almost
8990/// certainly some *other* candidate slot's own agent, and once the tail of
8991/// the roster is exhausted there are no more untried agents for *this* seat
8992/// to fall through to — the caller's fallback chain ends there, exactly as
8993/// "no further untried agents remain in the list for that seat" asks for.
8994///
8995/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8996/// the same id twice (an operator's `roles.implementers` typo, or a
8997/// `[[agents]]` list reused across roles) must not let
8998/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8999/// over `roster` either finds an untried id or runs out, so this always
9000/// terminates regardless of duplicates.
9001fn next_untried_in_roster<'a>(
9002    roster: &'a [AgentSpec],
9003    start: usize,
9004    tried: &BTreeSet<String>,
9005) -> Option<&'a AgentSpec> {
9006    roster
9007        .get(start + 1..)?
9008        .iter()
9009        .find(|s| !tried.contains(&s.id))
9010}
9011
9012/// The successor for a seat, shared by all four seat kinds (implement, judge,
9013/// review, advise). `others` holds the ids that *currently* occupy the other
9014/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9015/// sees them), so roster entries beyond the seat count act as spares: a failed
9016/// seat goes to an agent no other seat holds whenever the roster permits.
9017///
9018/// `carried` is `Some` only for the review loop's carried failure history.
9019/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9020/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9021/// whole roster: untried and not another seat's occupant; (3) the plain walk
9022/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9023/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9024/// a better outcome than an empty seat, and it keeps the answer to "is there
9025/// a successor at all" exactly what it was before occupants were considered,
9026/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9027/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9028/// carried failure is a cheaper bet than doubling an agent on the panel, and
9029/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9030/// the same round are handled one at a time, so a seat later in the batch
9031/// sees an earlier one's new occupant but not yet its own freed agent.
9032fn pick_successor<'a>(
9033    roster: &'a [AgentSpec],
9034    start: usize,
9035    tried: &BTreeSet<String>,
9036    carried: Option<&BTreeSet<String>>,
9037    others: &BTreeSet<String>,
9038) -> Option<&'a AgentSpec> {
9039    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9040    roster
9041        .get(start + 1..)
9042        .and_then(|tail| {
9043            tail.iter()
9044                .filter(free)
9045                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9046        })
9047        .or_else(|| {
9048            carried
9049                .is_some()
9050                .then(|| roster.iter().find(free))
9051                .flatten()
9052        })
9053        .or_else(|| match carried {
9054            Some(c) => next_for_seat(roster, start, tried, c),
9055            None => next_untried_in_roster(roster, start, tried),
9056        })
9057}
9058
9059/// The next agent for a seat that carries its failure history across rounds
9060/// (the review loop). `round_tried` is this round's own bound and starts
9061/// empty every round; `carried_failed` only decides priority.
9062///
9063/// First: an id walking forward from `start`, never wrapping, that is neither
9064/// tried this round nor failed in an earlier one. Only when that is exhausted
9065/// does it rescue: the first roster id (in roster order, so this one *does*
9066/// look before `start`) not yet tried this round, which is by then a carried
9067/// failure. Each id is rescued at most once per round, so it cannot loop.
9068fn next_for_seat<'a>(
9069    roster: &'a [AgentSpec],
9070    start: usize,
9071    round_tried: &BTreeSet<String>,
9072    carried_failed: &BTreeSet<String>,
9073) -> Option<&'a AgentSpec> {
9074    roster
9075        .get(start + 1..)?
9076        .iter()
9077        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9078        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9079}
9080
9081/// Where a reviewer seat starts a round: the agent that last answered it when
9082/// it is still on the roster and not marked failed, else the spec's own agent
9083/// unless it failed, else the next roster agent that has not failed, else the
9084/// spec's own agent again (the whole roster failed). Ids no longer on the
9085/// roster are ignored. An empty roster has no handover, so the spec stands.
9086fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9087    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9088        return spec;
9089    };
9090    let ok = |id: &str| !h.failed.contains(id);
9091    if let Some(last) = h.last_ok.as_deref()
9092        && ok(last)
9093        && let Some(s) = roster.iter().find(|s| s.id == last)
9094    {
9095        return s.clone();
9096    }
9097    if ok(&spec.id) {
9098        return spec;
9099    }
9100    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9101    roster
9102        .get(from + 1..)
9103        .into_iter()
9104        .flatten()
9105        .chain(roster.iter())
9106        .find(|s| ok(&s.id))
9107        .cloned()
9108        .unwrap_or(spec)
9109}
9110
9111/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9112/// seed so a CLI that mints its session id up front (`--session-id`) never
9113/// reuses the uuid the previous agent already opened under the same seat key.
9114pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9115    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9116}
9117
9118/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9119/// so the classifier below and the code that builds the message cannot drift.
9120const TIMED_OUT: &str = "timed out";
9121
9122impl FailClass {
9123    /// `None` for an answer; otherwise how the turn failed.
9124    fn of(out: &AgentOutcome) -> Option<Self> {
9125        match out {
9126            AgentOutcome::Ok(_) => None,
9127            AgentOutcome::Quota(_) => Some(Self::Quota),
9128            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9129            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9130            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9131            // Not a failure of the agent. Callers check for it first; `None`
9132            // here means "no class to hand over on".
9133            AgentOutcome::ResourceBlocked(_) => None,
9134        }
9135    }
9136
9137    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9138    fn stem_word(&self) -> &'static str {
9139        match self {
9140            Self::Quota => "quota",
9141            _ => "handover",
9142        }
9143    }
9144}
9145
9146/// The message's first line with its variable parts removed — digit runs and
9147/// path-like tokens — so "exit code 2" and "exit code 7" read
9148/// as one kind of failure.
9149fn failure_signature(msg: &str) -> String {
9150    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9151    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9152    // the historical machine key while the process diagnosis uses readable prose.
9153    if let Some(code) = line
9154        .strip_prefix("process exited with no usable output (exit code ")
9155        .and_then(|s| s.strip_suffix(')'))
9156        .and_then(|s| s.parse::<i32>().ok())
9157    {
9158        line = format!("exited with some({code}) and no usable output");
9159    } else if matches!(
9160        line.as_str(),
9161        "process exited with no usable output (no exit code reported)"
9162            | "process exited with no usable output (exit code unavailable)"
9163    ) {
9164        line = "exited with none and no usable output".to_owned();
9165    }
9166    let mut out = Vec::new();
9167    for word in line.split_whitespace() {
9168        if word.contains('/') || word.contains('\\') {
9169            out.push("<path>".to_owned());
9170            continue;
9171        }
9172        let mut w = String::new();
9173        let mut in_digits = false;
9174        for c in word.chars() {
9175            if c.is_ascii_digit() {
9176                if !in_digits {
9177                    w.push('#');
9178                }
9179                in_digits = true;
9180            } else {
9181                in_digits = false;
9182                w.push(c);
9183            }
9184        }
9185        out.push(w);
9186    }
9187    out.join(" ").chars().take(120).collect()
9188}
9189
9190/// Whether a seat that just failed with `cur` may go to the next roster agent.
9191/// A quota or a timeout always may. Any other failure may not when the agent
9192/// before it failed the same way: an error the prompt causes would otherwise
9193/// walk the whole roster. `prev` is the class of the immediately preceding
9194/// agent's failure, so a quota or timeout in between breaks the run of
9195/// identical failures by itself.
9196fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9197    match cur {
9198        FailClass::Quota | FailClass::Timeout => true,
9199        FailClass::Other(_) => prev != Some(cur),
9200    }
9201}
9202
9203/// A short human reason for a failed outcome, for the handover record.
9204fn fail_reason(out: &AgentOutcome) -> String {
9205    match out {
9206        AgentOutcome::Ok(_) => String::new(),
9207        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9208        AgentOutcome::Dropped(o) => format!(
9209            "the CLI dropped the stream ({})",
9210            o.dropped
9211                .as_ref()
9212                .map(|d| d.why.as_str())
9213                .unwrap_or("it ended without delivering its answer")
9214        ),
9215        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9216            e.lines().next().unwrap_or("").chars().take(160).collect()
9217        }
9218    }
9219}
9220
9221/// Note one handover in the run: the structured record and, in the timeline,
9222/// the sentence a person reads. A quota keeps the wording it always had.
9223pub(crate) fn record_handover(
9224    state: &mut RunState,
9225    node: &str,
9226    seat: &str,
9227    from: &str,
9228    to: &str,
9229    class: &FailClass,
9230    reason: &str,
9231) {
9232    let message = if *class == FailClass::Quota {
9233        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9234    } else {
9235        format!("{seat}: handed over {from} -> {to} ({reason})")
9236    };
9237    state.event(node, message);
9238    state.handovers.push(Handover {
9239        at: Timestamp::now(),
9240        node: node.to_owned(),
9241        seat: seat.to_owned(),
9242        from: from.to_owned(),
9243        to: to.to_owned(),
9244        reason: reason.to_owned(),
9245    });
9246}
9247
9248/// Did this reply report running a command whose own CLI never confirmed an
9249/// exit status?
9250///
9251/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9252/// command *finished* (see that type's own doc), so this can only be `true`
9253/// for a command whose completion event carried no readable exit code — not
9254/// for one that simply is not mentioned at all. That is the one signal this
9255/// crate can read, from the same record `state.jobs` renders, about a reply
9256/// standing next to work its own CLI cannot vouch for finishing; it is
9257/// deliberately not a check on the exit code's *value* (a fixer legitimately
9258/// runs a command that fails mid-iteration before it succeeds) and not a
9259/// guess at a command still running in the background (which emits no event
9260/// at all, and so leaves no evidence here to find).
9261fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9262    commands.iter().any(|c| c.exit_code.is_none())
9263}
9264
9265/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9266/// trusted as a verified no-op — the adoption guard's own text-level half.
9267///
9268/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9269/// timed out): a marker only earns the benefit of the doubt from a turn the
9270/// CLI itself vouches for finishing properly, the same house style
9271/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9272/// *fix* report to for `commands`. A candidate that timed out, exited
9273/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9274/// is, whatever prose it wrote — this returns `None` before it ever looks at
9275/// `text`. The remaining guards (the tree really is empty, the evidence is
9276/// non-empty) are the caller's: this only reads what the reply *claimed*.
9277fn verified_noop_claim(
9278    usable: bool,
9279    commands: &[agent::CommandEvidence],
9280    text: &str,
9281) -> Option<String> {
9282    (usable && !has_unconfirmed_command(commands))
9283        .then(|| verdict::verified_noop(text))
9284        .flatten()
9285}
9286
9287fn short(commit: &str) -> String {
9288    commit.chars().take(7).collect()
9289}
9290
9291fn make_executable(path: &Path) -> Result<()> {
9292    #[cfg(unix)]
9293    {
9294        use std::os::unix::fs::PermissionsExt as _;
9295        let mut perms = std::fs::metadata(path)?.permissions();
9296        perms.set_mode(0o755);
9297        std::fs::set_permissions(path, perms)?;
9298    }
9299    #[cfg(not(unix))]
9300    {
9301        let _ = path;
9302    }
9303    Ok(())
9304}
9305
9306/// What every seat in one batch shares: where the answers are attributed, the
9307/// prompt overlay they inherit, and the build cache they are told to use.
9308///
9309/// A struct rather than four more parameters: `wave` also needs the run's
9310/// state (to record who is answering right now) and the attempt number, and
9311/// eight positional arguments is both unreadable and a clippy error.
9312struct WaveCtx<'a> {
9313    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9314    /// paid for it.
9315    run: &'a str,
9316    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9317    node: &'a str,
9318    prompts: &'a Prompts,
9319    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9320    cache: Option<&'a Path>,
9321    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9322    /// `JobRecord::round`. `None` for every other node.
9323    round: Option<usize>,
9324    /// Carry each seat's failed-agent history across waves (the review loop
9325    /// only): start-of-round priority and handover choice read
9326    /// [`RunState::seat_history`], and every answer or failure writes it.
9327    carry_seats: bool,
9328    /// The run's park / interrupt requests, so a wait for a shared resource
9329    /// can be abandoned the moment the operator asks to park.
9330    pauses: Option<(&'a Pause, &'a Pause)>,
9331}
9332
9333/// Shared scheduling policy for operations and spawned seat invocations.
9334fn repository_wait(
9335    state: &RunState,
9336    pauses: Option<(&Pause, &Pause)>,
9337) -> crate::repo_lock::WaitPolicy {
9338    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9339    let inherited = crate::repo_lock::WaitPolicy::current();
9340    crate::repo_lock::WaitPolicy::new(
9341        Duration::from_secs(state.config.graph.timeout_resource_wait),
9342        move || {
9343            inherited.cancelled()
9344                || pauses
9345                    .as_ref()
9346                    .is_some_and(|(a, b)| a.parked() || b.parked())
9347        },
9348    )
9349}
9350
9351fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9352    if error.is::<crate::repo_lock::ResourceBlocked>() {
9353        state.resource_block = Some(crate::run::ResourceBlock {
9354            cause: crate::run::ResourceBlockCause::Other,
9355            node: "repository".into(),
9356            kind: if error.to_string().contains("disk pressure") {
9357                "disk pressure"
9358            } else {
9359                "repository coordination"
9360            }
9361            .into(),
9362            detail: error.to_string(),
9363            at: Timestamp::now(),
9364        });
9365        state.event("resource", error.to_string());
9366    }
9367}
9368
9369async fn coordinate_operation(
9370    state: &mut RunState,
9371    cwd: &Path,
9372    wait: &crate::repo_lock::WaitPolicy,
9373) -> Result<Option<crate::repo_lock::SeatLease>> {
9374    let result =
9375        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9376            .await;
9377    if let Err(e) = &result {
9378        record_repository_block(state, e);
9379    }
9380    result
9381}
9382
9383/// Run one job, honouring the parallelism budget.
9384async fn run_one(
9385    job: SeatJob,
9386    sem: Arc<Semaphore>,
9387    ctx: &WaveCtx<'_>,
9388    state: &mut RunState,
9389    attempt: usize,
9390) -> (SeatState, AgentOutcome) {
9391    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9392        .await
9393        .pop()
9394        .expect("one job in, one result out");
9395    (seat, out)
9396}
9397
9398/// Run every job concurrently, capped by the semaphore, preserving order.
9399///
9400/// Every seat in the batch is recorded into [`RunState::active`] before the
9401/// wave starts and cleared as each answer lands, so the run's own record says
9402/// who is still being waited on rather than only who finished.
9403async fn wave(
9404    jobs: Vec<SeatJob>,
9405    sem: Arc<Semaphore>,
9406    ctx: &WaveCtx<'_>,
9407    state: &mut RunState,
9408    attempt: usize,
9409) -> Vec<(usize, SeatState, AgentOutcome)> {
9410    let WaveCtx {
9411        run,
9412        node,
9413        prompts,
9414        cache,
9415        round,
9416        carry_seats: _,
9417        pauses,
9418    } = *ctx;
9419    for job in &jobs {
9420        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9421    }
9422    if let Err(e) = state.save() {
9423        // A failed persist of "who is answering right now" must not abort the
9424        // wave: the seats are already being asked, and the alternative is
9425        // losing the answers to save a status line nobody may even be
9426        // watching.
9427        tracing::warn!("could not persist in-progress seats: {e:#}");
9428    }
9429    // Hold the shared build cache's lease for the whole batch, not per job:
9430    // several candidates (an implement wave) or a fixer legitimately share
9431    // one cache concurrently within this run, and that stays untouched — a
9432    // single lease taken once for the whole wave and released once it is
9433    // done is what stops a *different* borrower (another run's own wave, its
9434    // e2e/gate, a human's `magi review`) from interleaving a build into the
9435    // same directory while this one is in flight. Best-effort, not
9436    // all-or-nothing: a wave that cannot get the lease within its own
9437    // longest job's budget still runs — an hour of paid implementer calls is
9438    // not thrown away over cache contention — but every write-allowed seat
9439    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9440    // below), the same fallback a read-only seat always gets, rather than
9441    // building into a directory this run was never granted. The identity
9442    // record is still invalidated below either way, so the next tracked
9443    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9444    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9445    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9446    // never borrowed from a job's agent budget (and never charged to it either,
9447    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9448    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9449    let cache_guard = if let Some(cache_dir) = cache {
9450        if jobs_had_a_writer {
9451            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9452            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9453            match acquire_cache_lease(
9454                state,
9455                cache_dir,
9456                &owner,
9457                resource_limit,
9458                node,
9459                node,
9460                &cancel,
9461            )
9462            .await
9463            {
9464                Ok(Some(g)) => Some(g),
9465                // Parking: leave without spawning anything. The run parks at
9466                // the next boundary and the seats are asked afresh on resume.
9467                Ok(None) => {
9468                    if let Some(block) = &mut state.resource_block {
9469                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9470                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9471                        };
9472                    }
9473                    return block_wave(
9474                        jobs,
9475                        state,
9476                        node,
9477                        attempt,
9478                        "parked while waiting for the shared build cache",
9479                    );
9480                }
9481                // An unreadable lease is not contention: keep the old
9482                // fallback (no shared cache for this wave's seats).
9483                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9484                Err(e) => {
9485                    // Writing seats may not build into a cache this run does
9486                    // not hold, and an agent that cannot be given one is not
9487                    // asked at all: the run stops resumably instead.
9488                    let why = format!("{e:#}");
9489                    state.event(
9490                        "resource",
9491                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9492                    );
9493                    return block_wave(jobs, state, node, attempt, &why);
9494                }
9495            }
9496        } else {
9497            None
9498        }
9499    } else {
9500        None
9501    };
9502    // Every job keeps its whole budget: the wait above was not the agent's.
9503    // Restart the seats' clocks so "started" means "being spawned now".
9504    state.seats_spawning(node, attempt);
9505    if let Err(e) = state.save() {
9506        tracing::warn!("could not persist the start of a wave: {e:#}");
9507    }
9508    // Queueing behind this wave's own seats for a parallelism slot is normal
9509    // (each of them is bounded by its own agent timeout); only a wait beyond
9510    // all of them plus the resource limit means the slots never came back.
9511    let slot_limit = resource_limit
9512        + jobs
9513            .iter()
9514            .map(|j| j.timeout)
9515            .max()
9516            .unwrap_or_default()
9517            .saturating_mul(jobs.len() as u32);
9518    let mut set = tokio::task::JoinSet::new();
9519    // Seats announce themselves here the moment they hold a parallelism
9520    // permit, i.e. just before their CLI is spawned.
9521    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9522    let overlay = prompts.overlay(node);
9523    let repository_wait = repository_wait(state, pauses);
9524    for (i, mut job) in jobs.into_iter().enumerate() {
9525        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9526        if cache.is_some() {
9527            job.prompt.push('\n');
9528            job.prompt
9529                .push_str(&prompt::build_cache_note(node, job.allow_write));
9530        }
9531        let sem = Arc::clone(&sem);
9532        let run = run.to_owned();
9533        let node = node.to_owned();
9534        // Only implementers were told about the task's attachments, so only
9535        // their seats get the directory widened for reading.
9536        let attachments = if node == "implement" {
9537            state.attachments.clone()
9538        } else {
9539            Vec::new()
9540        };
9541        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9542        // `prompt::build_cache_note`'s doc for why setting it anyway is
9543        // exactly how a sandboxed reviewer's write refusal got reported as a
9544        // defect in the patch, not a property of its own seat. And a
9545        // write-allowed one is handed it only when the lease above was
9546        // actually acquired: a wave that could not get it (`cache_guard` is
9547        // `None`, see its own comment) must not send seats to build into a
9548        // directory this run does not hold - that is the exact concurrent,
9549        // unmanaged-write race this module exists to prevent, not something
9550        // "proceeding anyway" is allowed to reintroduce.
9551        let cache = cache
9552            .filter(|_| job.allow_write && cache_guard.is_some())
9553            .map(Path::to_path_buf);
9554        let spawned_tx = spawned_tx.clone();
9555        let repository_wait = repository_wait.clone();
9556        let coordination = state.config.coordination.clone();
9557        let fetched_base = state.fetched_base.clone();
9558        set.spawn(async move {
9559            // A wait for a parallelism slot is scheduling, not the agent: it
9560            // is bounded by its own limit and ends as a resource block, never
9561            // as a timeout of the agent that was not started.
9562            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9563                Ok(p) => p,
9564                Err(_) => {
9565                    return (
9566                        i,
9567                        job.seat,
9568                        AgentOutcome::ResourceBlocked(format!(
9569                            "no parallel slot became free within {}s",
9570                            slot_limit.as_secs()
9571                        )),
9572                        false,
9573                    );
9574                }
9575            };
9576            spawned_tx.send(job.seat.key.clone()).ok();
9577            let mut seat = job.seat;
9578            let out = repository_wait
9579                .scope(agent::invoke(
9580                    &job.spec,
9581                    &mut seat,
9582                    &Invocation {
9583                        scope: crate::agent::InvocationScope::Repository,
9584                        coordination: coordination.as_ref(),
9585                        fetched_base: fetched_base.as_ref(),
9586                        cwd: &job.cwd,
9587                        prompt: &job.prompt,
9588                        timeout: job.timeout,
9589                        allow_write: job.allow_write,
9590                        unsandboxed: false,
9591                        sessions: job.sessions,
9592                        artifacts: &job.artifacts,
9593                        stem: &job.stem,
9594                        run: &run,
9595                        node: &node,
9596                        cache_dir: cache.as_deref(),
9597                        attachments: &attachments,
9598                        writable: &[],
9599                    },
9600                ))
9601                .await;
9602            let interrupted = out
9603                .as_ref()
9604                .err()
9605                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9606            let out = match out {
9607                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9608                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9609                // Billed work the CLI failed to hand over is not an ordinary
9610                // failure, but its text is the CLI's raw error JSON, not an
9611                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9612                // read it as one by forgetting to check. `usable()` is always
9613                // false here (dropped implies an empty response), so this has
9614                // to be checked before the catch-all `Failed` below or the
9615                // one shape this exists for is lost with the rest.
9616                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9617                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9618                Ok(o) => AgentOutcome::Failed(format!(
9619                    "process exited with no usable output ({})",
9620                    crate::notices::exit_status(o.exit_code)
9621                )),
9622                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9623                    AgentOutcome::ResourceBlocked(e.to_string())
9624                }
9625                Err(e) => AgentOutcome::Failed(e.to_string()),
9626            };
9627            (i, seat, out, interrupted)
9628        });
9629    }
9630    drop(spawned_tx);
9631    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9632    loop {
9633        let joined = tokio::select! {
9634            Some(key) = spawned_rx.recv() => {
9635                // The clock of this seat starts now, not when it queued.
9636                state.seat_spawned(&key);
9637                if let Err(e) = state.save() {
9638                    tracing::warn!("could not persist a seat's start: {e:#}");
9639                }
9640                continue;
9641            }
9642            joined = set.join_next() => joined,
9643        };
9644        let Some(joined) = joined else {
9645            break;
9646        };
9647        let (i, seat, out, interrupted) = match joined {
9648            Ok(v) => v,
9649            // No seat to clear: a panicked task never reported which one it
9650            // was. The defensive sweep below this loop is what stops that
9651            // seat's `active` entry from surviving forever.
9652            Err(e) => {
9653                tracing::error!("agent task panicked: {e}");
9654                continue;
9655            }
9656        };
9657        state.seat_finished(&seat.key);
9658        if interrupted
9659            && node == "implement"
9660            && let Some(label) = state
9661                .candidates
9662                .iter()
9663                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9664                .map(|candidate| candidate.label)
9665        {
9666            state.candidate_preservation.insert(
9667                label,
9668                "interrupted implementation awaiting persistence".into(),
9669            );
9670            state.interrupted_implementations.insert(label);
9671        }
9672        if let AgentOutcome::ResourceBlocked(why) = &out
9673            && state.resource_block.is_none()
9674        {
9675            state.resource_block = Some(crate::run::ResourceBlock {
9676                cause: crate::run::ResourceBlockCause::Other,
9677                node: node.to_owned(),
9678                kind: if why.contains("disk pressure") {
9679                    "disk pressure"
9680                } else if why.contains("repository coordination") {
9681                    "repository coordination"
9682                } else {
9683                    "parallel slot"
9684                }
9685                .to_owned(),
9686                detail: why.clone(),
9687                at: Timestamp::now(),
9688            });
9689            let activity = if interrupted {
9690                "agent execution interrupted"
9691            } else {
9692                "no agent was started"
9693            };
9694            state.event("resource", format!("{node}: {activity} - {why}"));
9695        }
9696        record_jobs(state, node, round, &seat.key, &out);
9697        if let Err(e) = state.save() {
9698            tracing::warn!("could not persist a seat's completion: {e:#}");
9699        }
9700        if collected.len() <= i {
9701            collected.resize_with(i + 1, || None);
9702        }
9703        collected[i] = Some((i, seat, out));
9704    }
9705    // Belt-and-braces for the panic branch above: every seat this exact batch
9706    // started shares this `(node, attempt)` pair, and every seat that finished
9707    // normally already cleared itself, so anything left tagged with it here
9708    // can only be a panicked task's leftover. Cleared unconditionally rather
9709    // than left to read as still answering forever.
9710    if state
9711        .active
9712        .values()
9713        .any(|a| a.node == node && a.attempt == attempt)
9714    {
9715        state
9716            .active
9717            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9718        if let Err(e) = state.save() {
9719            tracing::warn!("could not persist the end of a wave: {e:#}");
9720        }
9721    }
9722    // Whether or not the lease above was actually held, several worktrees
9723    // may just have built into the cache with nothing here able to name one
9724    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9725    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9726    // clean it might not have strictly needed; trusting a stale match would
9727    // cost it a wrong answer.
9728    if let Some(cache_dir) = cache
9729        && jobs_had_a_writer
9730    {
9731        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9732    }
9733    if let Some(guard) = cache_guard {
9734        guard.release();
9735    }
9736    collected.into_iter().flatten().collect()
9737}
9738
9739/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9740/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9741/// this is data collection, not the fix-specific completion contract in
9742/// [`Runner::continue_fix_report`], and applies regardless of which node
9743/// asked.
9744///
9745/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9746/// evidence from; `Failed` does not, and correctly contributes nothing — a
9747/// timeout or crash is not itself evidence about a command the seat may have
9748/// started.
9749fn record_jobs(
9750    state: &mut RunState,
9751    node: &str,
9752    round: Option<usize>,
9753    seat: &str,
9754    out: &AgentOutcome,
9755) {
9756    let commands: &[agent::CommandEvidence] = match out {
9757        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9758        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9759    };
9760    let checked_at = Timestamp::now();
9761    for c in commands {
9762        state.jobs.push(JobRecord {
9763            node: node.to_owned(),
9764            round,
9765            seat: seat.to_owned(),
9766            id: c.id.clone(),
9767            description: c.description.clone(),
9768            checked_at,
9769            status: match c.exit_code {
9770                Some(0) => JobStatus::Completed,
9771                Some(_) => JobStatus::Failed,
9772                None => JobStatus::Unknown,
9773            },
9774            exit_code: c.exit_code,
9775            result_summary: c.result_summary.clone(),
9776            source: c.source.clone(),
9777        });
9778    }
9779}
9780
9781/// Is a review round clean, given how many reviewer seats answered against
9782/// how many the round expected?
9783///
9784/// A seat that never answered (timeout, crash, unparsable output) is not a
9785/// seat that read the patch and found nothing — treating it as such is
9786/// exactly the bug this function exists to close. Under the default `block`
9787/// policy a missing seat can never be clean; `warn` still requires the seats
9788/// that *did* answer to have found nothing blocking and verification to be
9789/// green.
9790///
9791/// `quota_missing` narrows that `block` default for exactly one cause of
9792/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9793/// a session limit lifts by the very next round buys nothing — the seat is
9794/// asked again with the same quota — so once every missing seat is accounted
9795/// for by a quota loss (and at least one seat *did* answer, so a decision has
9796/// something to rest on) the round is decided on the panel that could answer,
9797/// same as `warn` would. A panel that lost every seat to quota is not
9798/// decided here: `answered == 0` falls through to the existing `block`
9799/// fallback so a fully collapsed panel still waits rather than landing on no
9800/// review at all.
9801fn round_is_clean(
9802    blocking: usize,
9803    e2e_ok: bool,
9804    answered: usize,
9805    expected: usize,
9806    quota_missing: usize,
9807    policy: IncompleteReviewPolicy,
9808) -> bool {
9809    if blocking != 0 || !e2e_ok {
9810        return false;
9811    }
9812    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9813        return true;
9814    }
9815    answered > 0 && expected - answered <= quota_missing
9816}
9817
9818/// The review loop's own conclusion, derived entirely from its persisted
9819/// round records and the round budget that produced them — never from
9820/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9821/// recomputes the identical answer regardless of what an earlier node in the
9822/// same walk, or a previous walk, did to `status`.
9823///
9824/// `None` while more rounds remain to try, including when review never ran
9825/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9826/// gone clean, or the budget is spent, or the tree has stopped moving (see
9827/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9828///
9829/// - An incomplete panel that raised nothing is missing input, not a
9830///   verified tree — never a hand-off candidate, whatever verification said
9831///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9832/// - Otherwise, green e2e on the last round hands off (see
9833///   [`Runner::stop_reviewing`]); red e2e blocks.
9834///
9835/// A last round whose own verification is still `ResourceBlocked` — magi
9836/// itself never got a command to run, not evidence the patch is broken —
9837/// is neither: this returns `None` for it too, the same as "more rounds
9838/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9839/// handling of that shape) instead of this cheap recomputation guessing a
9840/// verdict a real attempt never produced.
9841pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9842    if max_rounds == 0
9843        || reviews.last().is_some_and(|r| {
9844            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9845        })
9846    {
9847        return Some(RunStatus::Gating);
9848    }
9849    let last = reviews.last()?;
9850    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9851    if reviews.len() < max_rounds && !stagnant {
9852        return None;
9853    }
9854    if last.incomplete() && last.blocking == 0 {
9855        return Some(RunStatus::Blocked);
9856    }
9857    if last.e2e_status() == E2eStatus::ResourceBlocked
9858        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9859    {
9860        return None;
9861    }
9862    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9863        RunStatus::Gating
9864    } else {
9865        RunStatus::Blocked
9866    })
9867}
9868
9869/// How long a re-ask may take, given the budget the first attempt had.
9870///
9871/// A `nudged` retry is a request to restate an answer the seat has already
9872/// worked out: it carries no new work, so it does not deserve the original
9873/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9874/// 133 seconds while a third sat for over ten minutes on a resumed session
9875/// holding 410 KB of prior output - and because the retry had inherited the
9876/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9877/// judging round whose other seats were long finished.
9878///
9879/// A quarter of the budget, with a floor so that a deliberately short timeout
9880/// does not collapse to nothing. A retry that re-sends the whole prompt
9881/// (because the seat kept no context) is the original job again, and keeps the
9882/// original budget.
9883pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9884    if nudged {
9885        (full / 4).max(Duration::from_secs(120)).min(full)
9886    } else {
9887        full
9888    }
9889}
9890
9891/// Run a wave and parse each reply, re-asking the seats whose reply was
9892/// unusable.
9893///
9894/// The re-ask is a nudge rather than the whole prompt again when the seat still
9895/// holds its conversation, which is the difference between a cheap retry and
9896/// paying for the entire candidate set twice.
9897///
9898/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9899/// successor in `roster` is handed to it instead of being re-asked: the
9900/// handover is the retry. A seat with no successor left (a single-agent
9901/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9902/// the asks to one seat in one node number at most
9903/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9904/// is asked once (a dropped stream is nudged first), and only the last agent
9905/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9906/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9907/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9908/// other failure stops the chain when the previous agent failed the same way
9909/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9910/// [`has_context`] is false and the job's own full prompt and full budget are
9911/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9912/// intermediate ones are not losses) and is returned as a failure like any
9913/// other absent seat — the caller decides whether the panel still has a
9914/// quorum. An empty `roster` disables handover: failures are nudged as they
9915/// always were, and a quota is simply lost. A reply that fails to parse or
9916/// validate is the prompt's doing and is only ever nudged, never handed over.
9917///
9918/// The returned [`SeatState`] names the agent that answered (or tried last).
9919#[allow(clippy::too_many_arguments)]
9920async fn ask_json_wave<T>(
9921    jobs: Vec<SeatJob>,
9922    sem: Arc<Semaphore>,
9923    retries: usize,
9924    roster: &[AgentSpec],
9925    ctx: &WaveCtx<'_>,
9926    losses: &mut Vec<QuotaLoss>,
9927    state: &mut RunState,
9928    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9929) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9930where
9931    T: serde::de::DeserializeOwned + Send + 'static,
9932{
9933    ask_wave_with(
9934        jobs,
9935        sem,
9936        retries,
9937        roster,
9938        ctx,
9939        losses,
9940        state,
9941        &|text: &str| {
9942            let v = verdict::extract_json::<T>(text)?;
9943            validate(&v)?;
9944            Ok(v)
9945        },
9946    )
9947    .await
9948}
9949
9950/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9951/// a node whose answer is prose (deliberation) shares the same handover,
9952/// failure classification, quota bookkeeping and bounds instead of a copy.
9953///
9954/// A seat handed to another roster agent is sent the job's `handover` prompt
9955/// (when it has one) rather than `prompt`: the new agent has no session, so a
9956/// resume-style prompt would be incomplete. That holds for the handover ask
9957/// and for every nudge to that agent whose `has_context` is false.
9958#[allow(clippy::too_many_arguments)]
9959async fn ask_wave_with<T>(
9960    jobs: Vec<SeatJob>,
9961    sem: Arc<Semaphore>,
9962    retries: usize,
9963    roster: &[AgentSpec],
9964    ctx: &WaveCtx<'_>,
9965    losses: &mut Vec<QuotaLoss>,
9966    state: &mut RunState,
9967    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9968) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9969where
9970    T: Send + 'static,
9971{
9972    let n = jobs.len();
9973    let originals: Vec<SeatJob> = jobs;
9974    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9975    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9976    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9977    // answer, N once it has gone through N nudges. Read back once this
9978    // returns, so a caller building a history record (`ReviewRecord`) can
9979    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9980    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9981    // field's own doc.
9982    let mut nudges: Vec<usize> = vec![0; n];
9983    // The agent now occupying each seat, the ids it has already been through,
9984    // where in the roster the walk began, the class of the last failure, and
9985    // the stem word of a handover not yet asked (full prompt, full budget).
9986    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9987    let mut tried: Vec<BTreeSet<String>> = specs
9988        .iter()
9989        .map(|s| BTreeSet::from([s.id.clone()]))
9990        .collect();
9991    let starts: Vec<usize> = specs
9992        .iter()
9993        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9994        .collect();
9995    let carry = ctx.carry_seats && !roster.is_empty();
9996    // Carried across rounds: ids that failed the seat earlier, and how the
9997    // last failure went (so a repeat of it is not handed over again).
9998    let carried: Vec<BTreeSet<String>> = originals
9999        .iter()
10000        .map(|j| {
10001            state
10002                .seat_history
10003                .get(&j.seat.key)
10004                .filter(|_| carry)
10005                .map(|h| h.failed.clone())
10006                .unwrap_or_default()
10007        })
10008        .collect();
10009    let mut prev: Vec<Option<FailClass>> = originals
10010        .iter()
10011        .map(|j| {
10012            state
10013                .seat_history
10014                .get(&j.seat.key)
10015                .filter(|_| carry)
10016                .and_then(|h| h.last_fail.clone())
10017        })
10018        .collect();
10019    let next_agent =
10020        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10021            let others: BTreeSet<String> = specs
10022                .iter()
10023                .enumerate()
10024                .filter(|(j, _)| *j != i)
10025                .map(|(_, s)| s.id.clone())
10026                .collect();
10027            pick_successor(
10028                roster,
10029                starts[i],
10030                tried,
10031                carry.then(|| &carried[i]),
10032                &others,
10033            )
10034            .cloned()
10035        };
10036    let mut fresh: Vec<Option<String>> = vec![None; n];
10037    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10038    let mut pending: Vec<usize> = (0..n).collect();
10039
10040    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10041    // asks: an agent with a successor is asked once and handed over, and only
10042    // a seat with no successor spends `retries` nudges on the same agent. This
10043    // only guarantees the loop's own termination whatever those say.
10044    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10045    for round in 0..max_rounds {
10046        if pending.is_empty() {
10047            break;
10048        }
10049        let mut batch = Vec::with_capacity(pending.len());
10050        let mut renudged: Vec<&str> = Vec::new();
10051        for &i in &pending {
10052            let src = &originals[i];
10053            // A seat now held by another agent than the job named has no
10054            // session of its own: it gets the full-context prompt whenever
10055            // it is asked in full (the handover ask, a nudge it cannot
10056            // resume).
10057            let full: &str = match &src.handover {
10058                Some(h) if specs[i].id != src.spec.id => h,
10059                _ => &src.prompt,
10060            };
10061            // The prompt and the budget are one decision: a nudge restates
10062            // finished work, a re-sent prompt redoes it.
10063            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10064                (
10065                    full.to_owned(),
10066                    src.timeout,
10067                    format!("{}-{word}-{}", src.stem, specs[i].id),
10068                )
10069            } else if nudges[i] == 0 {
10070                (full.to_owned(), src.timeout, src.stem.clone())
10071            } else {
10072                renudged.push(src.seat.key.as_str());
10073                let why = done[i]
10074                    .as_ref()
10075                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10076                    .unwrap_or_else(|| "no parsable answer".to_owned());
10077                let nudge = prompt::nudge(&why);
10078                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10079                let prompt = if nudged {
10080                    nudge
10081                } else {
10082                    format!("{full}\n\n---\n\n{nudge}")
10083                };
10084                (
10085                    prompt,
10086                    retry_budget(src.timeout, nudged),
10087                    format!("{}-retry{}", src.stem, nudges[i]),
10088                )
10089            };
10090            batch.push(SeatJob {
10091                spec: specs[i].clone(),
10092                seat: seats[i].clone(),
10093                cwd: src.cwd.clone(),
10094                prompt,
10095                timeout,
10096                allow_write: src.allow_write,
10097                sessions: src.sessions,
10098                artifacts: src.artifacts.clone(),
10099                stem,
10100                handover: None,
10101            });
10102        }
10103
10104        if !renudged.is_empty() {
10105            state.event(
10106                ctx.node,
10107                format!("retry {round}: re-asking {}", renudged.join(", ")),
10108            );
10109        }
10110        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10111        let mut still = Vec::new();
10112        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10113            seats[i] = seat;
10114            // No agent was asked: nothing to hand over, nudge, remember or
10115            // charge. The caller sees `resource_block` and stops the node.
10116            if let AgentOutcome::ResourceBlocked(why) = &out {
10117                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10118                continue;
10119            }
10120            let class = FailClass::of(&out);
10121            // A dropped stream is nudged first (the conversation is still
10122            // there to pick up); only a seat whose nudges are spent hands over.
10123            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10124                && nudges[i] < retries
10125                && !roster.is_empty();
10126            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10127                let next = should_hand_over(prev[i].as_ref(), &cur)
10128                    .then(|| next_agent(i, &tried[i], &specs))
10129                    .flatten();
10130                if carry {
10131                    let h = state
10132                        .seat_history
10133                        .entry(originals[i].seat.key.clone())
10134                        .or_default();
10135                    h.failed.insert(specs[i].id.clone());
10136                    h.last_fail = Some(cur.clone());
10137                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10138                        h.last_ok = None;
10139                    }
10140                    // Saved before the next agent is asked, so a restart in
10141                    // between does not forget who failed.
10142                    if let Err(e) = state.save() {
10143                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10144                    }
10145                }
10146                if let Some(next) = next {
10147                    record_handover(
10148                        state,
10149                        ctx.node,
10150                        &originals[i].seat.key,
10151                        &specs[i].id,
10152                        &next.id,
10153                        &cur,
10154                        &fail_reason(&out),
10155                    );
10156                    tried[i].insert(next.id.clone());
10157                    prev[i] = Some(cur.clone());
10158                    seats[i] =
10159                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10160                    specs[i] = next;
10161                    fresh[i] = Some(cur.stem_word().to_owned());
10162                    nudges[i] = 0;
10163                    done[i] = Some(Err(anyhow::anyhow!(
10164                        "handed over after: {}",
10165                        fail_reason(&out)
10166                    )));
10167                    still.push(i);
10168                    continue;
10169                }
10170            }
10171            if carry
10172                && !nudge_first
10173                && let Some(cur) = class.clone()
10174            {
10175                // The chain ended here (no successor, or a repeated failure).
10176                let h = state
10177                    .seat_history
10178                    .entry(originals[i].seat.key.clone())
10179                    .or_default();
10180                h.failed.insert(specs[i].id.clone());
10181                h.last_fail = Some(cur);
10182            }
10183            let parsed = match out {
10184                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10185                AgentOutcome::Quota(o) => {
10186                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10187                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10188                }
10189                // Not a parseable answer: the nudge loop re-asks it, which is
10190                // exactly what a dropped stream needs. Just don't hand its raw
10191                // error JSON to `extract_json`.
10192                AgentOutcome::Dropped(o) => {
10193                    let why = o
10194                        .dropped
10195                        .as_ref()
10196                        .map(|d| d.why.as_str())
10197                        .unwrap_or("the CLI ended the stream without delivering its answer");
10198                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10199                }
10200                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10201                    Err(anyhow::anyhow!(e))
10202                }
10203            };
10204            let quota = class == Some(FailClass::Quota);
10205            let failed = parsed.is_err();
10206            done[i] = Some(parsed);
10207            if carry && !failed {
10208                let h = state
10209                    .seat_history
10210                    .entry(originals[i].seat.key.clone())
10211                    .or_default();
10212                h.failed.remove(&specs[i].id);
10213                h.last_ok = Some(specs[i].id.clone());
10214                h.last_fail = None;
10215            }
10216            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10217            // fail the same way; and never re-ask a seat that already parsed.
10218            // A failed agent that still has a successor is not re-asked
10219            // either: the handover was its remedy and has just been refused
10220            // (the chain stops on a repeated failure class). A seat with no
10221            // successor left (a single-agent roster, the roster's tail, or an
10222            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10223            let agent_failure = class.is_some()
10224                && !nudge_first
10225                && !roster.is_empty()
10226                && next_agent(i, &tried[i], &specs).is_some();
10227            if failed && !quota && !agent_failure && nudges[i] < retries {
10228                nudges[i] += 1;
10229                still.push(i);
10230            }
10231        }
10232        pending = still;
10233    }
10234
10235    // One loss per seat whose chain ended on a quota: the intermediate ones
10236    // were absorbed by a handover and are not losses.
10237    for (i, q) in last_quota.into_iter().enumerate() {
10238        if let Some(reset) = q {
10239            losses.push(QuotaLoss {
10240                seat: originals[i].seat.key.clone(),
10241                node: ctx.node.to_owned(),
10242                at: Timestamp::now(),
10243                reset,
10244            });
10245        }
10246    }
10247
10248    seats
10249        .into_iter()
10250        .zip(done)
10251        .zip(nudges)
10252        .map(|((seat, res), attempts)| {
10253            (
10254                seat,
10255                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10256                attempts,
10257            )
10258        })
10259        .collect()
10260}
10261
10262/// Did the wave end without asking any agent (see [`block_wave`])?
10263fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10264    results
10265        .iter()
10266        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10267}
10268
10269/// Settle a wave that never spawned anything: every job answers
10270/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10271/// else about the seat (session, history, quota) is touched.
10272fn block_wave(
10273    jobs: Vec<SeatJob>,
10274    state: &mut RunState,
10275    node: &str,
10276    attempt: usize,
10277    why: &str,
10278) -> Vec<(usize, SeatState, AgentOutcome)> {
10279    state.resource_wait = None;
10280    state
10281        .active
10282        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10283    if let Err(e) = state.save() {
10284        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10285    }
10286    jobs.into_iter()
10287        .enumerate()
10288        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10289        .collect()
10290}
10291
10292/// Acquire the shared build cache's lease, waiting out contention within
10293/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10294/// unbounded wait is never acceptable).
10295///
10296/// A first, non-blocking check happens before ever waiting; if it finds the
10297/// lease busy, that fact is logged as a `verify` event *and* flushed with
10298/// [`RunState::save`] immediately — not only once the wait finally succeeds
10299/// or gives up — so a `magi show` run by a different process while this one
10300/// is still waiting reads a `run.json` that says so, rather than whatever it
10301/// looked like before the wait started. The same applies to the terminal
10302/// failure: logged and saved before this returns `Err`, so a caller that
10303/// could not get the lease at all still leaves a legible record of why.
10304async fn acquire_cache_lease(
10305    state: &mut RunState,
10306    cache_dir: &Path,
10307    owner: &crate::cache::Owner,
10308    budget: Duration,
10309    node: &str,
10310    context: &str,
10311    cancel: &(dyn Fn() -> bool + Sync),
10312) -> Result<Option<crate::cache::Guard>> {
10313    let home = crate::run::home();
10314    let started = Instant::now();
10315    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10316        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10317        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10318        Err(e) => {
10319            state.event(
10320                "verify",
10321                format!("{context}: could not check the shared build cache: {e:#}"),
10322            );
10323            if let Err(e2) = state.save() {
10324                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10325            }
10326            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10327        }
10328    };
10329    state.event(
10330        "verify",
10331        format!(
10332            "{context}: waiting for the shared build cache at {} ({})",
10333            cache_dir.display(),
10334            busy.describe()
10335        ),
10336    );
10337    // Seats of this node are waiting, not running: say so, so the countdown a
10338    // reader sees only ever measures an agent that has been spawned.
10339    state.resource_wait = Some(crate::run::ResourceWait {
10340        node: context.to_owned(),
10341        kind: "shared build cache".to_owned(),
10342        holder: busy.describe(),
10343        since: Timestamp::now(),
10344        limit_secs: budget.as_secs(),
10345    });
10346    if let Err(e) = state.save() {
10347        tracing::warn!("could not persist a cache wait: {e:#}");
10348    }
10349    let remaining = budget.saturating_sub(started.elapsed());
10350    let waited = crate::cache::wait_for_unless(
10351        &home,
10352        cache_dir,
10353        owner,
10354        remaining,
10355        Duration::from_secs(5),
10356        cancel,
10357    )
10358    .await;
10359    state.resource_wait = None;
10360    let cause = match &waited {
10361        Ok(Some(_)) => None,
10362        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10363        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10364            Some(crate::run::ResourceBlockCause::Timeout {
10365                waited_secs: started.elapsed().as_secs(),
10366                limit_secs: budget.as_secs(),
10367            })
10368        }
10369        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10370    };
10371    if let Some(cause) = cause {
10372        let detail = match &waited {
10373            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10374            Err(error) => format!("{error:#}"),
10375        };
10376        state.resource_block = Some(crate::run::ResourceBlock {
10377            cause,
10378            node: node.to_owned(),
10379            kind: "shared build cache".into(),
10380            detail,
10381            at: Timestamp::now(),
10382        });
10383        state.event(
10384            "resource",
10385            state.resource_block.as_ref().unwrap().description(),
10386        );
10387    }
10388    if let Err(error) = state.save() {
10389        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10390    }
10391    waited
10392}
10393
10394/// The lease state could not be read at all (unreadable lease file, a
10395/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10396/// known to be contended, so a wave degrades to running without a shared cache
10397/// instead of stopping on a hold that [`crate::triage`] could never release.
10398#[derive(Debug)]
10399struct CacheCheckError(String);
10400
10401impl std::fmt::Display for CacheCheckError {
10402    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10403        f.write_str(&self.0)
10404    }
10405}
10406
10407impl std::error::Error for CacheCheckError {}
10408
10409/// Keep a specific acquisition outcome when the caller adds command evidence.
10410fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10411    state
10412        .resource_block
10413        .get_or_insert_with(|| crate::run::ResourceBlock {
10414            cause: crate::run::ResourceBlockCause::Other,
10415            node: node.to_owned(),
10416            kind: "shared build cache".into(),
10417            detail: format!("{error:#}"),
10418            at: Timestamp::now(),
10419        });
10420    if let Err(error) = state.save() {
10421        tracing::warn!("could not persist cache resource block: {error:#}");
10422    }
10423}
10424
10425/// Run `body` — a verify command batch — while holding the shared build
10426/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10427/// never interleave with another borrower's build against the same
10428/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10429/// timeout, or a human's own `magi review`. See the `cache` module doc for
10430/// why this matters more than Cargo's own per-target locking covers — two
10431/// *different* worktrees building the same package name/version into one
10432/// cache directory is a staleness bug, not a lock contention one.
10433///
10434/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10435/// agent waves do. Repository scheduling also has its own bounded,
10436/// cancellable resource-wait policy. Contention must not consume the
10437/// verification budget: `body` receives the full `budget` after both leases
10438/// are acquired and the cache is fresh. Without a shared cache, only
10439/// repository coordination is acquired before handing over the full budget.
10440///
10441/// A lease that cannot be acquired within the resource wait is reported as a single
10442/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10443/// verification — the same shape a spawn failure already takes in
10444/// [`run_commands`], so a caller need not special-case it.
10445#[allow(clippy::too_many_arguments)]
10446async fn with_cache_lease<'s, F, Fut>(
10447    state: &'s mut RunState,
10448    cache_dir: Option<&Path>,
10449    node: &str,
10450    seat: &str,
10451    worktree: &Path,
10452    head: &str,
10453    budget: Duration,
10454    context: &str,
10455    wait: &crate::repo_lock::WaitPolicy,
10456    body: F,
10457) -> Result<(Vec<CommandOutcome>, bool)>
10458where
10459    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10460    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10461{
10462    // Match editing waves: cache first, repository second. In particular,
10463    // pre_gate must never hold the repository while waiting for a wave's cache.
10464    let started = Instant::now();
10465    let home = crate::run::home();
10466    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10467    let guard = if let Some(cache_dir) = cache_dir {
10468        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10469        match acquire_cache_lease(
10470            state,
10471            cache_dir,
10472            &owner,
10473            resource_limit,
10474            node,
10475            context,
10476            &|| wait.cancelled(),
10477        )
10478        .await
10479        .and_then(|g| {
10480            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10481        }) {
10482            Ok(g) => Some(g),
10483            Err(e) => {
10484                record_cache_error(state, node, &e);
10485                return Ok((
10486                    vec![CommandOutcome {
10487                        command: "(waiting for the shared build cache)".into(),
10488                        code: None,
10489                        output_tail: e.to_string(),
10490                        duration_ms: started.elapsed().as_millis() as u64,
10491                        resource_blocked: true,
10492                    }],
10493                    false,
10494                ));
10495            }
10496        }
10497    } else {
10498        None
10499    };
10500    let _coordination = coordinate_operation(state, worktree, wait).await?;
10501    if let Some(cache_dir) = cache_dir {
10502        let identity = crate::cache::Identity::new(worktree, head);
10503        if let Err(e) = crate::cache::ensure_fresh(
10504            &home,
10505            cache_dir,
10506            &identity,
10507            &state.config.verify.cargo_manifests(),
10508        ) {
10509            record_cache_error(state, node, &e);
10510            state.event(
10511                "verify",
10512                format!(
10513                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10514                    worktree.display(),
10515                    short(head)
10516                ),
10517            );
10518            if let Some(guard) = guard {
10519                guard.release();
10520            }
10521            return Ok((
10522                vec![CommandOutcome {
10523                    command: "(confirming the shared build cache is fresh)".into(),
10524                    code: None,
10525                    output_tail: e.to_string(),
10526                    duration_ms: started.elapsed().as_millis() as u64,
10527                    resource_blocked: true,
10528                }],
10529                false,
10530            ));
10531        }
10532    }
10533    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10534    if !timed_out_pids.is_empty() {
10535        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10536    }
10537    // Drop repository coordination before releasing the cache, reversing the
10538    // acquisition order. A cancelled/error future also drops both guards.
10539    drop(_coordination);
10540    if let Some(guard) = guard {
10541        guard.release();
10542    }
10543    Ok((outcomes, retried))
10544}
10545
10546/// Poll `pids` — commands [`run_commands`] reports as still running when its
10547/// own timeout elapsed — until every one is confirmed gone, or
10548/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10549///
10550/// Real confirmation where confirmation is possible, not a substitute for
10551/// full process-tree observation: a grandchild the timed-out process spawned
10552/// and that survives independently of it is invisible to a pid check the
10553/// same way it always was, and continuing to observe and collect *that*
10554/// stays a different piece of work with its own owner. This only narrows a
10555/// fixed blind wait into an actual check of the pids this process does know
10556/// about.
10557async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10558    wait_for_pids_with(
10559        pids,
10560        crate::proc::pid_alive,
10561        LEASE_RELEASE_POLL,
10562        LEASE_RELEASE_MAX_WAIT,
10563    )
10564    .await;
10565}
10566
10567/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10568/// interval and ceiling supplied by the caller, so the polling *logic* -
10569/// returns as soon as every pid reports dead, gives up at the ceiling
10570/// otherwise - is testable on millisecond durations without asking the real
10571/// OS about a pid at all.
10572async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10573    pids: &[u32],
10574    alive: F,
10575    poll: Duration,
10576    max_wait: Duration,
10577) {
10578    let deadline = Instant::now() + max_wait;
10579    loop {
10580        if pids.iter().all(|&pid| !alive(pid)) {
10581            return;
10582        }
10583        if Instant::now() >= deadline {
10584            return;
10585        }
10586        tokio::time::sleep(poll).await;
10587    }
10588}
10589
10590/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10591/// admission that it could not even get a verify command to run, as opposed
10592/// to evidence the command actually produced? A caller that would otherwise
10593/// read a resource-blocked outcome as a red command must check this first:
10594/// see [`Runner::gate`], which retries rather than records `Blocked` when
10595/// this is true.
10596fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10597    outcomes.iter().any(|o| o.resource_blocked)
10598}
10599
10600/// What [`Runner::gate_fix_round`] decided.
10601pub(crate) enum GateFix {
10602    /// The tree changed and `verify.e2e` is still green: run the gate again.
10603    Retry,
10604    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10605    /// last failure stands and the run ends blocked.
10606    Stop,
10607    /// `verify.e2e` could not run after the fix (magi's own contention):
10608    /// decide nothing now, a later reentry retries.
10609    Defer,
10610}
10611
10612/// Is every red command in `outcomes` an ordinary failure the code could
10613/// explain: it ran, exited non-zero, and said something?
10614///
10615/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10616/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10617/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10618/// what the command is stays the gate's business.
10619fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10620    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10621    red.peek().is_some()
10622        && red.all(|o| {
10623            !o.resource_blocked
10624                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10625                && !o.output_tail.trim().is_empty()
10626        })
10627}
10628
10629/// Describe one verify command's outcome for the event log, distinguishing a
10630/// build/link failure — the toolchain never produced a binary to run — from
10631/// an actual test failure, since only the latter is a verdict on the patch.
10632fn e2e_outcome_label(o: &CommandOutcome) -> String {
10633    if o.ok() {
10634        return "pass".to_owned();
10635    }
10636    let reason = if o.build_failed() {
10637        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10638    } else {
10639        format!("FAIL ({:?})", o.code)
10640    };
10641    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10642}
10643
10644/// Run `verify.e2e`, retrying once if the first attempt could not build or
10645/// link — a build/link failure is frequently a race against a shared
10646/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10647/// `verify` event per command, tagged with `context` (normally `"round N"`)
10648/// so the two call sites that need this — the ordinary per-round leg in
10649/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10650/// it will ever call a round green — read identically in the event log.
10651async fn run_e2e_with_retry(
10652    state: &mut RunState,
10653    shell: &[String],
10654    commands: &[String],
10655    worktree: &Path,
10656    timeout: Duration,
10657    context: &str,
10658) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10659    let (mut e2e, mut timed_out_pids) = run_commands(
10660        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10661    )
10662    .await;
10663    for o in &e2e {
10664        state.event(
10665            "verify",
10666            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10667        );
10668    }
10669    // A build/link failure is not a verdict on the patch — it is frequently a
10670    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10671    // one retry before letting a red like that decide the round.
10672    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10673    if verify_retried {
10674        state.event(
10675            "verify",
10676            format!(
10677                "{context}: verify could not build/link, not a test result — retrying once \
10678                 before concluding"
10679            ),
10680        );
10681        let retried = run_commands(
10682            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10683        )
10684        .await;
10685        e2e = retried.0;
10686        // Both attempts' timeouts matter, not just the last one: the first
10687        // attempt's descendants may still be alive alongside the retry's.
10688        timed_out_pids.extend(retried.1);
10689        for o in &e2e {
10690            state.event(
10691                "verify",
10692                format!(
10693                    "{context}: retry `{}` -> {}",
10694                    o.command,
10695                    e2e_outcome_label(o)
10696                ),
10697            );
10698        }
10699    }
10700    (e2e, verify_retried, timed_out_pids)
10701}
10702
10703/// Run configured shell commands in `cwd`, in order. The second element is
10704/// the pid of every command that hit `timeout` and was still running when
10705/// this stopped waiting on it (best-effort: `None` when the platform did not
10706/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10707/// that releases a shared resource afterward needs to know.
10708///
10709/// Records `task` into [`RunState::active`] at every command boundary
10710/// (`RunState::task_command`) and clears it once the whole list has run
10711/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10712/// for minutes with no seat and no output of its own to show for it (see
10713/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10714/// apart from "not yet run" without this), and this is the only place that
10715/// knows which command is running right now and how many are left. Three
10716/// saves per command — start, not per second — matching the same "only at a
10717/// boundary" rule [`wave`] already follows for seats.
10718#[allow(clippy::too_many_arguments)]
10719async fn run_commands(
10720    state: &mut RunState,
10721    node: &str,
10722    task: &str,
10723    attempt: usize,
10724    shell: &[String],
10725    commands: &[String],
10726    cwd: &Path,
10727    timeout: Duration,
10728) -> (Vec<CommandOutcome>, Vec<u32>) {
10729    if commands.is_empty() {
10730        // Nothing to mark as running and nothing to clear — an empty list
10731        // means "not configured", and touching `active` (or the disk) over
10732        // that would be a write for every round of a repo with no
10733        // `verify.e2e` / `verify.gate` commands at all.
10734        return (Vec::new(), Vec::new());
10735    }
10736    let mut out = Vec::new();
10737    let mut timed_out_pids = Vec::new();
10738    let total = commands.len();
10739    for (idx, command) in commands.iter().enumerate() {
10740        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10741        if let Err(e) = state.save() {
10742            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10743        }
10744        let started = Instant::now();
10745        let mut cmd = tokio::process::Command::new(&shell[0]);
10746        cmd.quiet();
10747        // Verify the checkout as a standalone caller. Worker ownership and
10748        // pinned executable selection belong to this driver, not to tests or
10749        // tools it runs. Preserve home/seat context and Running's containment.
10750        for key in crate::worker::INVOCATION_ENV {
10751            cmd.env_remove(key);
10752        }
10753        cmd.args(&shell[1..])
10754            .arg(command)
10755            .current_dir(cwd)
10756            .stdin(std::process::Stdio::null())
10757            .stdout(std::process::Stdio::piped())
10758            .stderr(std::process::Stdio::piped())
10759            .kill_on_drop(true);
10760        let mut outputs = Vec::new();
10761        let protected = (|| -> Result<()> {
10762            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10763                outputs.push(guard);
10764            }
10765            if let Some(cache) = state.config.cache_dir() {
10766                outputs.push(crate::retention::OutputLease::cache(
10767                    &crate::run::home(),
10768                    &cache,
10769                    false,
10770                )?);
10771            }
10772            Ok(())
10773        })();
10774        if let Err(e) = protected {
10775            out.push(CommandOutcome {
10776                command: command.clone(),
10777                code: None,
10778                output_tail: format!("output ownership unavailable: {e:#}"),
10779                duration_ms: 0,
10780                resource_blocked: true,
10781            });
10782            continue;
10783        }
10784        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10785        let (code, body) = match spawned {
10786            Ok(mut running) => {
10787                let pid = running.child().id();
10788                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10789                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10790                let mut paths = vec![cwd.to_owned()];
10791                if let Some(cache) = state.config.cache_dir() {
10792                    paths.push(cache);
10793                }
10794                let mut pressure = None;
10795                let status = tokio::select! {
10796                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10797                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10798                        pressure = Some(reason); None
10799                    }
10800                };
10801                let (code, failure) = match status {
10802                    Some(Ok(Ok(status))) => (
10803                        status.code(),
10804                        status
10805                            .code()
10806                            .is_none()
10807                            .then(|| format!("command terminated: {status}")),
10808                    ),
10809                    Some(Ok(Err(error))) => {
10810                        (None, Some(format!("failed waiting for command: {error}")))
10811                    }
10812                    Some(Err(_)) => {
10813                        // Leased cleanup confirms process-tree death itself.
10814                        // Unleased commands still need the caller's PID check
10815                        // after kill-on-drop, before releasing cache ownership.
10816                        if !running.coordinated()
10817                            && let Some(pid) = pid
10818                        {
10819                            timed_out_pids.push(pid);
10820                        }
10821                        (
10822                            None,
10823                            Some(format!("timed out after {}s", timeout.as_secs())),
10824                        )
10825                    }
10826                    None => (None, None),
10827                };
10828                running.finish().await;
10829                if let Some(reason) = pressure {
10830                    state.resource_block = Some(crate::run::ResourceBlock {
10831                        cause: crate::run::ResourceBlockCause::Other,
10832                        node: node.to_owned(),
10833                        kind: "disk pressure".to_owned(),
10834                        detail: reason.clone(),
10835                        at: Timestamp::now(),
10836                    });
10837                    out.push(CommandOutcome {
10838                        command: command.clone(),
10839                        code: None,
10840                        output_tail: reason,
10841                        duration_ms: started.elapsed().as_millis() as u64,
10842                        resource_blocked: true,
10843                    });
10844                    break;
10845                }
10846                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10847                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10848                if let Some(failure) = failure {
10849                    body.push('\n');
10850                    body.push_str(&failure);
10851                }
10852                (code, body)
10853            }
10854            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10855        };
10856        out.push(CommandOutcome {
10857            command: command.clone(),
10858            code,
10859            output_tail: tail(&body, OUTPUT_TAIL),
10860            duration_ms: started.elapsed().as_millis() as u64,
10861            resource_blocked: false,
10862        });
10863    }
10864    state.task_finished(task);
10865    if let Err(e) = state.save() {
10866        tracing::warn!("could not persist the end of {task}: {e:#}");
10867    }
10868    (out, timed_out_pids)
10869}
10870
10871/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10872/// section (`report::run`) and in the `merge` event this node records — for
10873/// the operator to run by hand.
10874///
10875/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10876/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10877/// merge commits", or "require linear history") rejects the push a `--no-ff`
10878/// merge would produce, which is exactly the guidance this function replaces.
10879/// `message`'s first line becomes the squash commit's subject, matching the
10880/// note `report::run` prints alongside this command — see that function for
10881/// why an explicit subject is not optional there.
10882fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10883    let repo = repo.display();
10884    match style {
10885        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10886        MergeStyle::Squash => {
10887            // The subject sits inside double quotes, and a title an agent
10888            // wrote may carry the characters that break out of them.
10889            let subject = message
10890                .lines()
10891                .next()
10892                .unwrap_or(branch)
10893                .replace(['\\', '"', '$', '`'], "");
10894            format!(
10895                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10896            )
10897        }
10898        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10899    }
10900}
10901
10902/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10903/// under the hood, rejects a `title` over 256 characters and the whole
10904/// command fails — no PR at all, for a run whose body was otherwise fine
10905/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10906/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10907/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10908/// this bound. It is a margin, not a guarantee — a title packed
10909/// with multi-unit characters could still in principle land close to the
10910/// edge, but a real task title's occasional emoji or accented letter fits
10911/// comfortably inside it.
10912const PR_TITLE_MAX: usize = 240;
10913
10914/// A pull request title from the opening line of `text`, or `None` when that
10915/// line is not English (GitHub text is) or has no letters.
10916///
10917/// A line within `max` is kept as is. A longer one is cut at the end of its
10918/// first sentence when that falls inside `max`, else at a word boundary with a
10919/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10920/// make every merely-truncated title look non-English). The language check
10921/// runs on the kept text before any mark is added, so only what GitHub will
10922/// show is judged: an English opening followed by non-ASCII far past the cut
10923/// still passes.
10924fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10925    let line = queue::first_line(text)?;
10926    let chars: Vec<char> = line.chars().collect();
10927    let (kept, mark) = if chars.len() <= max {
10928        (line.to_owned(), "")
10929    } else if let Some(end) = sentence_end(&chars, max) {
10930        (chars[..end].iter().collect::<String>(), "")
10931    } else {
10932        let room = max.saturating_sub(3);
10933        // Cut at the last space inside the room; when the char just past the
10934        // room is a space the room already ends on a word.
10935        let cut = if chars[room].is_whitespace() {
10936            room
10937        } else {
10938            chars[..room]
10939                .iter()
10940                .rposition(|c| c.is_whitespace())
10941                .unwrap_or(room)
10942        };
10943        let head: String = chars[..cut].iter().collect();
10944        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10945        (head.to_owned(), "...")
10946    };
10947    if kept.is_empty() {
10948        return None;
10949    }
10950    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10951        return None;
10952    }
10953    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10954        return None;
10955    }
10956    Some(format!("{kept}{mark}"))
10957}
10958
10959/// The char length of the first sentence of `chars` when it ends within `max`
10960/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10961/// abbreviations so `e.g. foo` does not end a title early.
10962fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10963    const MIN: usize = 20;
10964    for i in MIN..max.min(chars.len()) {
10965        if !matches!(chars[i], '.' | '!' | '?') {
10966            continue;
10967        }
10968        let Some(&next) = chars.get(i + 1) else {
10969            continue;
10970        };
10971        if !next.is_whitespace() {
10972            continue;
10973        }
10974        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10975        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10976            continue;
10977        }
10978        let word: String = chars[..i]
10979            .iter()
10980            .rev()
10981            .take_while(|c| !c.is_whitespace())
10982            .collect::<Vec<_>>()
10983            .into_iter()
10984            .rev()
10985            .collect();
10986        let word = word.to_ascii_lowercase();
10987        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10988            continue;
10989        }
10990        let end = chars[..i]
10991            .iter()
10992            .rposition(|c| !c.is_whitespace())
10993            .map_or(i, |p| p + 1);
10994        return Some(end);
10995    }
10996    None
10997}
10998
10999/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11000/// change: a title and a body describing what was *implemented*, not the task
11001/// that asked for it. A task reads as a request; a reader of the merged
11002/// history wants the change.
11003struct PrMessage {
11004    title: String,
11005    body: String,
11006}
11007
11008impl PrMessage {
11009    /// Title, blank line, body. The first line is the squash/merge commit
11010    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11011    /// has to stay one sensible line.
11012    fn commit_message(&self) -> String {
11013        format!("{}\n\n{}", self.title, self.body)
11014    }
11015}
11016
11017/// The text after a leading `TITLE:` (any case) on `line`.
11018fn title_marker(line: &str) -> Option<&str> {
11019    let line = line.trim();
11020    let head = line.get(..6)?;
11021    head.eq_ignore_ascii_case("title:")
11022        .then(|| line[6..].trim())
11023}
11024
11025/// The implementer's own one-line title: the `TITLE:` line the implement
11026/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11027/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11028/// source, and a title that says as much is refused here too.
11029fn summary_title(summary: &str) -> Option<String> {
11030    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11031    let raw = title_marker(first)?;
11032    if raw.is_empty() {
11033        return None;
11034    }
11035    let title = queue::title_from(raw, PR_TITLE_MAX);
11036    let lower = title.to_ascii_lowercase();
11037    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11038        return None;
11039    }
11040    Some(title)
11041}
11042
11043/// How `open_review`'s instruction begins; see [`landing_title`].
11044const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11045
11046/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11047pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11048
11049/// Does this run review work that already existed, rather than implement a
11050/// task? Runs recorded before `reviewed_commits` existed carry only the
11051/// candidate marker.
11052pub(crate) fn is_review_run(state: &RunState) -> bool {
11053    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11054}
11055
11056/// The title of a review-only run: the subject of the oldest commit under
11057/// review. Later commits are usually fixups, and `instruction` is the review
11058/// prompt, which says nothing about the change. GitHub text is English, so a
11059/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11060fn review_title(state: &RunState) -> Option<String> {
11061    english_subject(
11062        state.reviewed_commits.as_ref()?.first()?,
11063        state.config.graph.github_english(),
11064    )
11065}
11066
11067/// `raw` as a pull request title, or `None` when it is blank, not English
11068/// (GitHub text is), or one of magi's own candidate commit subjects.
11069fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11070    let raw = raw.trim();
11071    if raw.is_empty() {
11072        return None;
11073    }
11074    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11075        return None;
11076    }
11077    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11078        return None;
11079    }
11080    let title = queue::title_from(raw, PR_TITLE_MAX);
11081    let lower = title.to_ascii_lowercase();
11082    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11083        return None;
11084    }
11085    Some(title)
11086}
11087
11088/// What a review-only run's branch says about itself, read at the moment the
11089/// pull request is opened.
11090#[derive(Debug, Clone, PartialEq, Eq)]
11091struct BranchFacts {
11092    /// `(subject, body)` of each commit, oldest first.
11093    commits: Vec<(String, String)>,
11094    /// Trimmed `git diff --stat`.
11095    stat: String,
11096}
11097
11098/// Longest diff stat shown: this many file lines plus the summary line.
11099const STAT_FILE_LINES: usize = 25;
11100/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11101const COMMITS_MAX_CHARS: usize = 20_000;
11102
11103/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11104/// say or finds nothing, so the caller falls back to what the run recorded.
11105async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11106    let commits = git::commit_log(repo, base, branch).await.ok()?;
11107    if commits.is_empty() {
11108        return None;
11109    }
11110    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11111    let lines: Vec<&str> = stat.lines().collect();
11112    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11113        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11114        let more = format!(" ... {omitted} more file(s)");
11115        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11116        kept.push(&more);
11117        kept.push(lines[lines.len() - 1]);
11118        kept.join("\n")
11119    } else {
11120        lines.join("\n")
11121    };
11122    Some(BranchFacts { commits, stat })
11123}
11124
11125/// Defang what would break the surrounding markdown: a closing `</details>`
11126/// and a code fence.
11127fn markdown_safe(text: &str) -> String {
11128    text.replace("</details>", "&lt;/details&gt;")
11129        .replace("\x60\x60\x60", "~~~")
11130}
11131
11132fn neutral_title(state: &RunState, winner: char) -> String {
11133    format!(
11134        "chore: land candidate {} of run {}",
11135        winner.to_ascii_uppercase(),
11136        state.id
11137    )
11138}
11139
11140/// Reuse the graph's bounded fixer without starting another graph or driver.
11141/// Copy the working state back even on errors so spent rounds remain visible.
11142pub(crate) async fn repair_landing_gate(
11143    state: &mut RunState,
11144    winner: &Candidate,
11145    outcomes: &[CommandOutcome],
11146) -> Result<GateFix> {
11147    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11148        return Ok(GateFix::Stop);
11149    }
11150    let mut runner = Runner {
11151        roles: state.config.resolve_roles()?,
11152        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11153        state: state.clone(),
11154        pause: Pause::new(),
11155        interrupt: Pause::new(),
11156    };
11157    let result = runner.gate_fix_round(winner, outcomes).await;
11158    *state = runner.state;
11159    result
11160}
11161
11162/// Recheck a repair's e2e requirement across failures, resource deferrals and
11163/// process interruptions. A successful result only applies to its exact SHA.
11164pub(crate) async fn check_gate_repair_e2e(
11165    state: &mut RunState,
11166    winner: &Candidate,
11167    wait: &crate::repo_lock::WaitPolicy,
11168) -> Result<GateFix> {
11169    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11170        return Ok(GateFix::Retry);
11171    }
11172    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11173    if state
11174        .gate_repair_e2e
11175        .as_ref()
11176        .is_some_and(|r| r.head == head && r.passed)
11177    {
11178        return Ok(GateFix::Retry);
11179    }
11180    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11181        head: head.clone(),
11182        passed: false,
11183        outcomes: Vec::new(),
11184    });
11185    state.save()?;
11186    let commands = state.config.verify.e2e.clone();
11187    let outcomes = if commands.is_empty() {
11188        Vec::new()
11189    } else {
11190        let shell = state.config.shell();
11191        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11192        let cache_dir = state.config.cache_dir();
11193        let (outcomes, _) = with_cache_lease(
11194            state,
11195            cache_dir.as_deref(),
11196            "e2e",
11197            "e2e",
11198            &winner.worktree,
11199            &head,
11200            timeout,
11201            "gate repair e2e",
11202            wait,
11203            |state, budget| async move {
11204                run_e2e_with_retry(
11205                    state,
11206                    &shell,
11207                    &commands,
11208                    &winner.worktree,
11209                    budget,
11210                    "gate repair e2e",
11211                )
11212                .await
11213            },
11214        )
11215        .await?;
11216        outcomes
11217    };
11218    let deferred = verify_inconclusive(&outcomes);
11219    let passed = outcomes.iter().all(CommandOutcome::ok)
11220        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11221    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11222        head,
11223        passed,
11224        outcomes,
11225    });
11226    state.save()?;
11227    if deferred {
11228        Ok(GateFix::Defer)
11229    } else if !passed {
11230        state.event(
11231            "gate",
11232            "verify.e2e failed after gate repair; nothing published",
11233        );
11234        Ok(GateFix::Stop)
11235    } else {
11236        Ok(GateFix::Retry)
11237    }
11238}
11239
11240/// Run the same configured gate for a landing head and a fresh graph.
11241pub(crate) async fn run_landing_gate(
11242    state: &mut RunState,
11243    winner: &Candidate,
11244    wait: &crate::repo_lock::WaitPolicy,
11245) -> Result<Vec<CommandOutcome>> {
11246    let shell = state.config.shell();
11247    let gate_commands = state.config.verify.gate.clone();
11248    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11249    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11250        Vec::new()
11251    } else {
11252        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11253        let cache_dir = state.config.cache_dir();
11254        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11255        let (outcomes, _) = with_cache_lease(
11256            state,
11257            cache_dir.as_deref(),
11258            "gate",
11259            "gate",
11260            &winner.worktree,
11261            &head,
11262            timeout,
11263            "final gate",
11264            wait,
11265            |state, budget| {
11266                let shell = shell.clone();
11267                let gate_commands = gate_commands.clone();
11268                let winner = winner.clone();
11269                async move {
11270                    Runner::run_pre_gate(state, &winner).await;
11271                    let (outcomes, timed_out_pids) = run_commands(
11272                        state,
11273                        "gate",
11274                        "gate",
11275                        0,
11276                        &shell,
11277                        &gate_commands,
11278                        &winner.worktree,
11279                        budget,
11280                    )
11281                    .await;
11282                    (outcomes, false, timed_out_pids)
11283                }
11284            },
11285        )
11286        .await?;
11287        outcomes
11288    };
11289    if outcomes.is_empty() {
11290        // Nothing configured to check — distinct from every other
11291        // silence in this run's event log, since an empty `gate` alone
11292        // no longer says whether the gate ran at all (see
11293        // `RunState::gate_ran`'s own doc).
11294        state.event("gate", "no gate commands configured; not tested");
11295    }
11296    for o in &outcomes {
11297        state.event(
11298            "gate",
11299            format!(
11300                "`{}` -> {}",
11301                o.command,
11302                if o.ok() {
11303                    "pass".to_owned()
11304                } else {
11305                    format!(
11306                        "FAIL ({:?})\n{}",
11307                        o.code,
11308                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11309                    )
11310                }
11311            ),
11312        );
11313    }
11314    Ok(outcomes)
11315}
11316
11317/// The pull request title to hand to `land::merge_subject`. A review-only run
11318/// opened by an earlier build titled its pull request with the review prompt;
11319/// that title is dropped (empty, so the fallback applies) rather than landed.
11320/// Any other title, including an operator's rename, passes through untouched,
11321/// and so does every title of a run that implements a task.
11322pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11323    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11324        ""
11325    } else {
11326        pr_title
11327    }
11328}
11329
11330/// What the squash subject falls back to when the pull request title is empty
11331/// or candidate-shaped: for a review-only run the derived title, never the
11332/// review prompt held in `instruction`.
11333pub fn landing_subject_source(state: &RunState) -> String {
11334    if is_review_run(state) {
11335        let winner = state.candidates.first().map_or('A', |c| c.label);
11336        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11337    }
11338    state.instruction.clone()
11339}
11340
11341/// `summary` without its `TITLE:` line, which the pull request title already
11342/// carries.
11343fn summary_without_title(summary: &str) -> String {
11344    let mut lines = summary.trim().lines().peekable();
11345    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11346        lines.next();
11347    }
11348    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11349}
11350
11351/// The pull request title and body for the winning candidate.
11352///
11353/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11354/// the task's own opening line via [`queue::title_from`] when there is none.
11355/// `state.instruction` can open with blank lines (`task_text` only rejects a
11356/// body that is blank *entirely*), which `title_from` skips.
11357///
11358/// Publishing uses an independent description generated from the final diff
11359/// and structured evidence. Internal summaries and reports remain local.
11360/// Non-publishing previews may display the original task and candidate summary.
11361/// The footer retains the run and candidate provenance.
11362#[cfg(test)]
11363fn pr_message(state: &RunState, winner: char) -> PrMessage {
11364    pr_message_with(state, winner, None)
11365}
11366
11367/// [`pr_message`] with what the branch of a review-only run says about itself.
11368/// `facts` is ignored for a run that implements a task.
11369#[cfg(test)]
11370fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11371    let raw = pr_message_raw(
11372        state,
11373        winner,
11374        facts,
11375        crate::pr_outcome::FinalHead::Unreported,
11376        ChangeNote::NotAsked,
11377    );
11378    let id = crate::scrub::Identity::current();
11379    PrMessage {
11380        title: crate::scrub::scrub(&raw.title, &id),
11381        body: crate::scrub::scrub(&raw.body, &id),
11382    }
11383}
11384
11385/// What the public description may say about the change itself.
11386#[derive(Debug, Clone, Copy)]
11387enum ChangeNote<'a> {
11388    /// A local preview or non-publishing merge mode: no public generation needed.
11389    NotAsked,
11390    /// A summary written from the final diff.
11391    Summary(&'a crate::run::ChangeSummary),
11392    /// A summary was asked for and could not be produced; the text says why.
11393    Unavailable(&'a str),
11394}
11395
11396fn pr_message_raw(
11397    state: &RunState,
11398    winner: char,
11399    facts: Option<&BranchFacts>,
11400    head: crate::pr_outcome::FinalHead<'_>,
11401    change: ChangeNote<'_>,
11402) -> PrMessage {
11403    if !matches!(change, ChangeNote::NotAsked) {
11404        let (title, mut body) = match change {
11405            ChangeNote::Summary(c) => (
11406                c.title.clone(),
11407                format!("## Summary\n\n{}\n\n", c.body.trim()),
11408            ),
11409            ChangeNote::Unavailable(why) => (
11410                neutral_title(state, winner),
11411                format!(
11412                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11413                ),
11414            ),
11415            ChangeNote::NotAsked => unreachable!(),
11416        };
11417        body.push_str(&public_pr_evidence(state, head));
11418        body.push_str(&format!(
11419            "\n---\nmagi:run/{} magi:candidate-{}\n",
11420            state.id,
11421            winner.to_ascii_lowercase()
11422        ));
11423        return PrMessage { title, body };
11424    }
11425    pr_preview(state, winner, facts, head)
11426}
11427
11428/// Append public facts from the run records after every prose generation,
11429/// including posting-gate rewrites. The agent cannot drop this provenance.
11430fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11431    let mut body = crate::pr_outcome::review_rounds(state);
11432    body.push_str(&crate::pr_outcome::final_verification(state, head));
11433    let open = state.last_round_findings();
11434    if !open.is_empty() {
11435        body.push_str("## Open review findings\n\n");
11436        for f in open {
11437            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11438        }
11439        body.push('\n');
11440    }
11441    if let Some(c) = state.change_summary.as_ref()
11442        && head == crate::pr_outcome::FinalHead::At(&c.head)
11443    {
11444        body.push_str(&format!(
11445            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11446            c.base, c.head
11447        ));
11448    }
11449    body
11450}
11451
11452/// Local-only preview of the operator's task and candidate summary.
11453fn pr_preview(
11454    state: &RunState,
11455    winner: char,
11456    facts: Option<&BranchFacts>,
11457    head: crate::pr_outcome::FinalHead<'_>,
11458) -> PrMessage {
11459    let summary = state
11460        .candidates
11461        .iter()
11462        .find(|c| c.label == winner)
11463        .map(|c| c.summary.as_str())
11464        .unwrap_or_default();
11465    // The fallback is the operator's own words and may not be English; GitHub
11466    // text always is, so a non-English task gets a neutral title instead.
11467    let review = is_review_run(state);
11468    let title = if review {
11469        facts
11470            .and_then(|f| {
11471                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11472            })
11473            .or_else(|| review_title(state))
11474            .or_else(|| {
11475                state
11476                    .candidates
11477                    .iter()
11478                    .find(|c| c.label == winner)
11479                    .filter(|c| !c.branch.starts_with("magi/"))
11480                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11481            })
11482            .unwrap_or_else(|| neutral_title(state, winner))
11483    } else {
11484        summary_title(summary).unwrap_or_else(|| {
11485            english_title(
11486                &state.instruction,
11487                PR_TITLE_MAX,
11488                state.config.graph.github_english(),
11489            )
11490            .unwrap_or_else(|| neutral_title(state, winner))
11491        })
11492    };
11493
11494    let mut body = String::new();
11495    let what = summary_without_title(summary);
11496    if !what.is_empty() {
11497        body.push_str("## Summary\n\n");
11498        body.push_str(&what);
11499        body.push_str("\n\n");
11500    }
11501
11502    // Review history uses structured facts across all rounds, never reports.
11503    body.push_str(&crate::pr_outcome::review_rounds(state));
11504    body.push_str(&crate::pr_outcome::final_verification(state, head));
11505    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11506
11507    let open = state.open_findings();
11508    if !open.is_empty() {
11509        body.push_str("## Open review findings\n\n");
11510        for f in &open {
11511            body.push_str(&format!(
11512                "- `{}` [{:?}] {}\n",
11513                f.id, f.severity, "Context retained in local review records."
11514            ));
11515        }
11516        body.push('\n');
11517    }
11518
11519    if let Some(fix) = fix
11520        && !fix.rejected.is_empty()
11521    {
11522        body.push_str("## Declined by the fixer\n\n");
11523        for r in &fix.rejected {
11524            body.push_str(&format!(
11525                "- `{}`: {}\n",
11526                r.id, "Reason retained in local review records."
11527            ));
11528        }
11529        body.push('\n');
11530    }
11531
11532    if review {
11533        // The review prompt is not the task; list what the branch carries.
11534        body.push_str("## Commits under review\n\n");
11535        if let Some(facts) = facts {
11536            let mut left = COMMITS_MAX_CHARS;
11537            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11538                let mut entry = format!("- {}\n", markdown_safe(subject));
11539                for l in markdown_safe(text).lines() {
11540                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11541                }
11542                if left == 0 {
11543                    body.push_str(&format!(
11544                        "- ... {} more commit(s)\n",
11545                        facts.commits.len() - i
11546                    ));
11547                    break;
11548                }
11549                if entry.len() > left {
11550                    // Even the first commit is cut: one huge body must not
11551                    // push the whole description past GitHub's limit.
11552                    let mut end = left;
11553                    while !entry.is_char_boundary(end) {
11554                        end -= 1;
11555                    }
11556                    entry.truncate(end);
11557                    entry.push_str("\n  ... (truncated)\n");
11558                    left = 0;
11559                } else {
11560                    left -= entry.len();
11561                }
11562                body.push_str(&entry);
11563            }
11564            if !facts.stat.trim().is_empty() {
11565                body.push_str(&format!(
11566                    "\n## Diff stat\n\n```\n{}\n```\n",
11567                    markdown_safe(facts.stat.trim())
11568                ));
11569            }
11570        } else {
11571            match &state.reviewed_commits {
11572                Some(subjects) => {
11573                    for s in subjects {
11574                        body.push_str(&format!("- {}\n", s.trim()));
11575                    }
11576                }
11577                None => {
11578                    // An older run kept only the prompt, with the commit list
11579                    // after its first paragraph.
11580                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11581                    body.push_str(rest.trim());
11582                    body.push('\n');
11583                }
11584            }
11585        }
11586    } else {
11587        let task = state.instruction.trim();
11588        let task = if task.is_empty() {
11589            "(empty task)"
11590        } else {
11591            task
11592        };
11593        body.push_str(&format!(
11594            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11595            task.replace("</details>", "&lt;/details&gt;")
11596        ));
11597    }
11598
11599    body.push_str(&format!(
11600        "\n---\nmagi:run/{} magi:candidate-{}\n",
11601        state.id,
11602        winner.to_ascii_lowercase()
11603    ));
11604
11605    PrMessage { title, body }
11606}
11607
11608/// Source imports have already been prepared from structured inputs. Legacy
11609/// inferred references must never instruct a seat to reconstruct those imports.
11610fn seeded_instruction(state: &RunState) -> String {
11611    let mut inputs = Vec::new();
11612    if let Some(start) = &state.sources.start_from {
11613        inputs.push(format!(
11614            "Start from `{}` at `{}` (including ancestry).",
11615            start.token, start.sha
11616        ));
11617    }
11618    for pick in &state.sources.pick_commits {
11619        inputs.push(format!(
11620            "Pick `{}` at `{}` (single commit).",
11621            pick.token, pick.sha
11622        ));
11623    }
11624    if inputs.is_empty() {
11625        state.instruction.clone()
11626    } else {
11627        format!(
11628            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11629            state.instruction,
11630            inputs.join("\n")
11631        )
11632    }
11633}
11634
11635/// Does the winner have no commits ahead of the base it would land on?
11636/// Any failure to find out reads as "not empty": the merge then behaves as it
11637/// always did rather than refusing on a guess.
11638async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11639    let base = &state.base_branch;
11640    let mut against = base.clone();
11641    if mode == MergeMode::Pr {
11642        // A pull request lands on the remote's base, never the local branch
11643        // of the same name: if that cannot be read, "not empty" is the safe
11644        // answer.
11645        let remote = &state.config.merge.remote;
11646        let tracking = format!("{remote}/{base}");
11647        let fetched = git::fetch(repo, remote, base).await;
11648        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11649            against = tracking;
11650        } else {
11651            return false;
11652        }
11653    }
11654    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11655}
11656
11657/// Why nothing was opened for an empty winner, with what the task's own
11658/// references resolved to.
11659fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11660    let mut detail = format!(
11661        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11662         nothing to open a pull request for"
11663    );
11664    detail.push_str(
11665        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11666    );
11667    detail
11668}
11669
11670/// What the `Pr` merge does once it knows whether the branch already has an
11671/// open pull request.
11672#[derive(Debug, PartialEq, Eq)]
11673enum PrPlan {
11674    Create,
11675    Adopt { url: String, title: String },
11676    Stop(String),
11677}
11678
11679/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11680/// a failed lookup -> stop with the real reason. Never guesses.
11681fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11682    match found {
11683        Ok(land::OpenPr::None) => PrPlan::Create,
11684        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11685        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11686            "several open pull requests exist for this branch, not picking one: {}",
11687            urls.join(" ")
11688        )),
11689        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11690    }
11691}
11692
11693/// `gh pr create`, returning the PR url.
11694pub(crate) async fn gh_pr_create(
11695    cwd: &Path,
11696    base: &str,
11697    head: &str,
11698    title: &str,
11699    body: &str,
11700) -> Result<String> {
11701    let out = tokio::process::Command::new("gh")
11702        .args([
11703            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11704        ])
11705        .current_dir(cwd)
11706        .quiet()
11707        .stdin(std::process::Stdio::null())
11708        .output()
11709        .await
11710        .context("spawn gh")?;
11711    if out.status.success() {
11712        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11713    } else {
11714        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11715    }
11716}
11717
11718/// Never force deletion of uncommitted work, including unknown legacy trees.
11719async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11720    match std::fs::symlink_metadata(path) {
11721        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11722            // A previous cleanup may have removed the tree before a branch
11723            // deletion or state save failed. Unregister it if still present.
11724            git::worktree_remove_clean(repo, path).await?;
11725            return Ok(());
11726        }
11727        Err(e) => return Err(e.into()),
11728        Ok(_) => {}
11729    }
11730    if !git::is_clean(path).await? {
11731        bail!(
11732            "worktree has uncommitted work; refusing removal: {}",
11733            path.display()
11734        );
11735    }
11736    if !git::worktree_remove_clean(repo, path).await? {
11737        bail!("worktree removal refused: {}", path.display());
11738    }
11739    Ok(())
11740}
11741
11742/// Tear a run's worktrees and branches down.
11743///
11744/// `home` is where the updated `run.json` is saved (via
11745/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11746/// a housekeeping pass already has its own honest `home` handed to it, and
11747/// falling through to the global here would write back through whichever
11748/// directory some other process or test pinned into that `OnceLock` first,
11749/// not the one the caller actually resolved its `runs` and `state` from.
11750pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11751    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11752        bail!("refusing fold: {reason}");
11753    }
11754    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11755        bail!("candidate preservation incomplete; refusing cleanup");
11756    }
11757    // Serialize ordinary folding with the same starts/resumes as target-only
11758    // eviction. Otherwise a second janitor could remove the whole worktree
11759    // around a live output borrower even though target eviction was fenced.
11760    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11761    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11762    let _driver = crate::driver::maintenance(home, &state.id)?;
11763    let mut output_guards = Vec::new();
11764    for entry in std::fs::read_dir(state.worktree_root())
11765        .into_iter()
11766        .flatten()
11767        .flatten()
11768    {
11769        if entry.path().join(".git").is_file() {
11770            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11771        }
11772    }
11773    let repo = state.repo.clone();
11774    // CLI, Web and automatic cleanup all enter here, independently of
11775    // Runner::fold_losers. Protect deletion and shared config release too.
11776    let wait = repository_wait(state, None);
11777    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11778        Ok(lease) => lease,
11779        Err(e) => {
11780            state.save_under(home)?;
11781            return Err(e);
11782        }
11783    };
11784    // Confirmed merges dispose of their owned sources after durable evidence
11785    // capture. Preflight every source before removing anything.
11786    let _source_outputs = if state.status == RunStatus::Merged
11787        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11788    {
11789        verify_merged_fold_sources(state, drop_winner, home).await?
11790    } else {
11791        Vec::new()
11792    };
11793    let root = state.worktree_root();
11794    let winner = state.tally.as_ref().map(|t| t.winner);
11795    let mut removed = Vec::new();
11796
11797    for i in 0..state.candidates.len() {
11798        let c = state.candidates[i].clone();
11799        let is_winner = Some(c.label) == winner;
11800        if is_winner && !drop_winner {
11801            continue;
11802        }
11803        let existed = c.worktree.exists();
11804        // A vanished directory can still have a Git registration that keeps
11805        // its branch checked out. Remove that registration before the branch.
11806        remove_preserved_worktree(&repo, &c.worktree).await?;
11807        if existed {
11808            removed.push(c.worktree.to_string_lossy().into_owned());
11809        }
11810        // A branch handed to a later run (and its pull request) is not this
11811        // run's to delete.
11812        let handed_over = state.released_branches.contains(&c.branch);
11813        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11814            if !git::branch_delete(&repo, &c.branch).await? {
11815                bail!("candidate branch removal refused");
11816            }
11817            removed.push(c.branch.clone());
11818        }
11819        state.candidates[i].folded = true;
11820    }
11821
11822    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11823        let path = name.path();
11824        let keep = !drop_winner
11825            && winner.is_some_and(|w| {
11826                path.file_name()
11827                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11828            });
11829        if keep {
11830            continue;
11831        }
11832        remove_preserved_worktree(&repo, &path).await?;
11833        removed.push(path.to_string_lossy().into_owned());
11834    }
11835
11836    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11837    // judge worktrees, so once the loop above has cleared all of them out,
11838    // the parent is a bare directory nobody else was ever going to remove -
11839    // git only ever managed what was inside it. Left alone, one of these
11840    // accumulates per fully-folded run; the operator's own machine had 74.
11841    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11842    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11843    // so does anything a run never claimed that happens to share the bay.
11844    remove_if_empty(&root);
11845
11846    if state.enabled_worktree_config && drop_winner {
11847        // A release, not a raw disable: some sibling run in this repository
11848        // may still hold its own reference (see `git::acquire_worktree_config`),
11849        // and only the last release actually turns the setting back off.
11850        git::release_worktree_config(&repo).await.ok();
11851        state.enabled_worktree_config = false;
11852    }
11853    state.save_under(home)?;
11854    Ok(removed)
11855}
11856
11857async fn verify_merged_fold_sources(
11858    state: &RunState,
11859    drop_winner: bool,
11860    home: &Path,
11861) -> Result<Vec<std::fs::File>> {
11862    let winner = state.tally.as_ref().map(|t| t.winner);
11863    let mut heads = BTreeSet::new();
11864    for candidate in &state.candidates {
11865        if Some(candidate.label) == winner && !drop_winner {
11866            continue;
11867        }
11868        anyhow::ensure!(
11869            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11870            "candidate recovery source is outside the inventoried run root: {}",
11871            candidate.worktree.display()
11872        );
11873        match std::fs::symlink_metadata(&candidate.worktree) {
11874            Ok(_) => {}
11875            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11876            Err(error) => return Err(error.into()),
11877        }
11878        if !state.released_branches.contains(&candidate.branch)
11879            && git::branch_exists(&state.repo, &candidate.branch).await?
11880        {
11881            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11882        }
11883    }
11884    let mut cargo_guards = Vec::new();
11885    let entries = match std::fs::read_dir(state.worktree_root()) {
11886        Ok(entries) => Some(entries),
11887        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11888        Err(error) => return Err(error.into()),
11889    };
11890    for entry in entries.into_iter().flatten() {
11891        let path = entry?.path();
11892        let metadata = std::fs::symlink_metadata(&path)?;
11893        anyhow::ensure!(
11894            metadata.is_dir() && !metadata.file_type().is_symlink(),
11895            "non-directory or symlink recovery source retained: {}",
11896            path.display()
11897        );
11898        if !drop_winner
11899            && winner.is_some_and(|w| {
11900                path.file_name()
11901                    .is_some_and(|name| name == format!("cand-{w}").as_str())
11902            })
11903        {
11904            continue;
11905        }
11906        anyhow::ensure!(
11907            path.join(".git").is_file(),
11908            "unregistered recovery source retained: {}",
11909            path.display()
11910        );
11911        anyhow::ensure!(
11912            git::git(
11913                &path,
11914                &[
11915                    "status",
11916                    "--porcelain",
11917                    "--untracked-files=all",
11918                    "--ignored",
11919                    "--",
11920                    ".",
11921                    ":(exclude)target"
11922                ]
11923            )
11924            .await?
11925            .is_empty(),
11926            "source changes or ignored recovery files retained: {}",
11927            path.display()
11928        );
11929        match std::fs::symlink_metadata(path.join("target")) {
11930            Ok(_) => {
11931                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11932            }
11933            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11934            Err(error) => return Err(error.into()),
11935        }
11936        for checkpoint in [
11937            "rebase-merge",
11938            "rebase-apply",
11939            "sequencer",
11940            "CHERRY_PICK_HEAD",
11941            "MERGE_HEAD",
11942            "REBASE_HEAD",
11943            "REVERT_HEAD",
11944            "BISECT_LOG",
11945            "AUTO_MERGE",
11946            "MERGE_MSG",
11947        ] {
11948            let location = git::git(
11949                &path,
11950                &[
11951                    "rev-parse",
11952                    "--path-format=absolute",
11953                    "--git-path",
11954                    checkpoint,
11955                ],
11956            )
11957            .await?;
11958            anyhow::ensure!(
11959                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11960                if e.kind() == std::io::ErrorKind::NotFound),
11961                "Git recovery checkpoint retained: {checkpoint}"
11962            );
11963        }
11964        heads.insert(git::rev_parse(&path, "HEAD").await?);
11965    }
11966    crate::fold_evidence::preserve(state, home, &heads).await?;
11967
11968    Ok(cargo_guards)
11969}
11970
11971/// Remove `dir` if it exists and has nothing in it.
11972///
11973/// Best-effort and silent by design: a directory that is not empty (a run
11974/// whose winner is still parked there, a stray file some other process left)
11975/// is exactly the case this must refuse, and a directory that is already gone
11976/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11977/// already refuses a non-empty directory, so the emptiness check below is
11978/// belt, not suspenders - it is what keeps this from ever attempting the
11979/// removal in the case that matters, rather than trusting `remove_dir`'s
11980/// error path to have no side effects if it ever changed.
11981fn remove_if_empty(dir: &Path) {
11982    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11983        std::fs::remove_dir(dir).ok();
11984    }
11985}
11986
11987/// Severity of the worst open finding in the last review round, for reporting.
11988pub fn worst_open(state: &RunState) -> Option<Severity> {
11989    state
11990        .reviews
11991        .last()?
11992        .reviews
11993        .iter()
11994        .flat_map(|r| r.findings.iter())
11995        .map(|f| f.severity)
11996        .max()
11997}
11998
11999/// `[worktree] setup` hides its products per worktree, which needs
12000/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12001/// worktrees, like the hook's; every place that runs setup calls this first.
12002async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12003    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12004        git::acquire_worktree_config(repo).await?;
12005        state.enabled_worktree_config = true;
12006    }
12007    Ok(())
12008}
12009
12010#[cfg(test)]
12011mod tests {
12012    #[tokio::test]
12013    async fn launcher_base_fetch_requires_remote_success() {
12014        let tmp = tempfile::tempdir().unwrap();
12015        let repo = tmp.path().join("repo");
12016        let remote = tmp.path().join("remote");
12017        std::fs::create_dir(&repo).unwrap();
12018        std::fs::create_dir(&remote).unwrap();
12019        git::git(&remote, &["init", "--bare", "-b", "main"])
12020            .await
12021            .unwrap();
12022        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12023        git::git(
12024            &repo,
12025            &[
12026                "-c",
12027                "user.name=Fixture",
12028                "-c",
12029                "user.email=fixture@example.invalid",
12030                "commit",
12031                "--allow-empty",
12032                "-m",
12033                "fixture",
12034            ],
12035        )
12036        .await
12037        .unwrap();
12038        git::git(
12039            &repo,
12040            &["remote", "add", "origin", remote.to_str().unwrap()],
12041        )
12042        .await
12043        .unwrap();
12044        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12045        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12046        assert_eq!(
12047            resolve_base(&repo, "main", "origin", None)
12048                .await
12049                .unwrap()
12050                .commit,
12051            head
12052        );
12053        #[cfg(unix)]
12054        {
12055            let policy: crate::config::Coordination = toml::from_str(
12056                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12057            ).unwrap();
12058            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12059            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12060            let held = crate::repo_lock::SeatLease::operation_with(
12061                &repo,
12062                Some(&policy),
12063                &crate::repo_lock::WaitPolicy::default(),
12064            )
12065            .await
12066            .unwrap();
12067            assert!(held.is_some());
12068            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12069            tokio::pin!(fetch);
12070            assert!(
12071                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12072                    .await
12073                    .is_err()
12074            );
12075            assert!(
12076                !repo.join(".git/FETCH_HEAD").exists(),
12077                "no fetch before admission"
12078            );
12079            drop(held);
12080            let fetched = fetch.await.unwrap();
12081            assert_eq!(fetched.commit, head);
12082            assert_eq!(fetched.remote, "origin");
12083            assert_eq!(fetched.branch, "main");
12084        }
12085        // A surviving tracking ref is not successful fetch evidence.
12086        git::git(
12087            &repo,
12088            &[
12089                "remote",
12090                "set-url",
12091                "origin",
12092                tmp.path().join("missing").to_str().unwrap(),
12093            ],
12094        )
12095        .await
12096        .unwrap();
12097        let error = resolve_base(&repo, "main", "origin", None)
12098            .await
12099            .unwrap_err()
12100            .to_string();
12101        assert!(error.contains("cannot read origin/main"));
12102        assert!(error.contains("refusing to branch off"));
12103    }
12104
12105    #[test]
12106    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12107        let leaked = vec!["chore(deps): update a crate".to_owned()];
12108        let own = "fix(daemon): apply a chosen action";
12109        assert!(should_retitle("", own, &leaked));
12110        assert!(should_retitle(
12111            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12112            own,
12113            &leaked
12114        ));
12115        assert!(should_retitle(
12116            "chore: land candidate A of run 1",
12117            own,
12118            &leaked
12119        ));
12120        assert!(should_retitle(
12121            "magi: candidate A (uncommitted work)",
12122            own,
12123            &leaked
12124        ));
12125        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12126        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12127        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12128    }
12129
12130    #[test]
12131    fn pr_merge_plan_creates_adopts_or_stops() {
12132        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12133        assert_eq!(
12134            pr_merge_plan(Ok(land::OpenPr::One {
12135                url: "u".into(),
12136                title: "t".into()
12137            })),
12138            PrPlan::Adopt {
12139                url: "u".into(),
12140                title: "t".into()
12141            }
12142        );
12143        let PrPlan::Stop(many) =
12144            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12145        else {
12146            panic!("many must stop");
12147        };
12148        assert!(many.contains('a') && many.contains('b'));
12149        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12150            panic!("a failed lookup must stop");
12151        };
12152        assert!(err.contains("bad token"));
12153    }
12154
12155    use super::*;
12156    use crate::run::GateStatus;
12157    use std::collections::BTreeMap;
12158    use std::time::Duration;
12159
12160    fn conductor() -> AgentSpec {
12161        AgentSpec {
12162            id: "conductor".to_owned(),
12163            kind: crate::config::AgentKind::Command,
12164            model: None,
12165            command: vec!["true".to_owned()],
12166            extra_args: Vec::new(),
12167            env: BTreeMap::new(),
12168            prompt_delivery: None,
12169            max_turn_secs: None,
12170        }
12171    }
12172
12173    fn spec(id: &str) -> AgentSpec {
12174        AgentSpec {
12175            id: id.to_owned(),
12176            kind: crate::config::AgentKind::Command,
12177            model: None,
12178            command: vec!["true".to_owned()],
12179            extra_args: Vec::new(),
12180            env: BTreeMap::new(),
12181            prompt_delivery: None,
12182            max_turn_secs: None,
12183        }
12184    }
12185
12186    fn ids(xs: &[&str]) -> BTreeSet<String> {
12187        xs.iter().map(|s| (*s).to_owned()).collect()
12188    }
12189
12190    #[test]
12191    fn pick_successor_skips_an_agent_another_seat_holds() {
12192        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12193        let roster = [spec("a"), spec("b"), spec("c")];
12194        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12195        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12196    }
12197
12198    #[test]
12199    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12200        // The other seat started on c but was handed to d; c is free again.
12201        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12202        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12203        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12204        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12205        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12206    }
12207
12208    #[test]
12209    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12210        let roster = [spec("a"), spec("b")];
12211        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12212        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12213        let carried = ids(&["b"]);
12214        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12215        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12216    }
12217
12218    #[test]
12219    fn pick_successor_returns_none_without_any_untried_successor() {
12220        let roster = [spec("a"), spec("b")];
12221        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12222        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12223        let carried = ids(&["a"]);
12224        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12225    }
12226
12227    #[test]
12228    fn pick_successor_rescue_avoids_another_seats_occupant() {
12229        // b is a carried failure and free; a is held by the other seat.
12230        let roster = [spec("a"), spec("b"), spec("c")];
12231        let carried = ids(&["b", "c"]);
12232        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12233        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12234    }
12235
12236    #[test]
12237    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12238        let roster = [spec("a"), spec("b"), spec("c")];
12239        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12240        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12241    }
12242
12243    #[test]
12244    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12245        let roster = [spec("a"), spec("b"), spec("c")];
12246        let failed = ids(&["a", "b", "c"]);
12247        // Rescue looks at the whole roster, once per id, then runs out.
12248        let mut tried = ids(&["b"]);
12249        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12250        assert_eq!(first.id, "a");
12251        tried.insert(first.id.clone());
12252        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12253        assert_eq!(second.id, "c");
12254        tried.insert(second.id.clone());
12255        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12256    }
12257
12258    #[test]
12259    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12260        let roster = [spec("a"), spec("b")];
12261        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12262        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12263    }
12264
12265    #[test]
12266    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12267        let roster = [spec("a"), spec("b"), spec("c")];
12268        let h = SeatHistory {
12269            failed: ids(&["a"]),
12270            last_ok: Some("b".to_owned()),
12271            last_fail: None,
12272        };
12273        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12274        // A last answerer that left the roster, or later failed, is ignored.
12275        let gone = SeatHistory {
12276            last_ok: Some("zzz".to_owned()),
12277            ..h.clone()
12278        };
12279        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12280        let failed = SeatHistory {
12281            failed: ids(&["a", "b"]),
12282            last_ok: Some("b".to_owned()),
12283            last_fail: None,
12284        };
12285        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12286    }
12287
12288    #[test]
12289    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12290        let first = SeatState::new("review-1", "alpha", 7);
12291        let next = handover_seat("review-1", "gamma", 7);
12292        assert_ne!(first.claude_session, next.claude_session);
12293    }
12294
12295    #[test]
12296    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12297        let mut state = state_with_summary("x", "y");
12298        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12299        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12300        assert_ne!(a.claude_session, b.claude_session);
12301    }
12302
12303    #[test]
12304    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12305        let roster = [spec("a"), spec("b")];
12306        let h = SeatHistory {
12307            failed: ids(&["a", "b"]),
12308            ..SeatHistory::default()
12309        };
12310        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12311        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12312        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12313    }
12314
12315    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12316    // fallback loop depends on to terminate: it must walk forward from the
12317    // seat's own position, never restart at the front of the roster, and it
12318    // must never hand back an id already tried, however many times that id
12319    // happens to appear.
12320
12321    #[test]
12322    fn failure_signature_ignores_numbers_and_paths() {
12323        assert_eq!(
12324            failure_signature("exited with Some(2) and no usable output"),
12325            failure_signature("exited with Some(137) and no usable output")
12326        );
12327        assert_eq!(
12328            failure_signature("process exited with no usable output (exit code 2)"),
12329            failure_signature("process exited with no usable output (exit code 137)")
12330        );
12331        for (old, new) in [
12332            (
12333                "exited with Some(2) and no usable output",
12334                "process exited with no usable output (exit code 2)",
12335            ),
12336            (
12337                "exited with None and no usable output",
12338                "process exited with no usable output (exit code unavailable)",
12339            ),
12340        ] {
12341            let previous = FailClass::Other(failure_signature(old));
12342            let current = FailClass::Other(failure_signature(new));
12343            assert_eq!(previous, current, "resumption retains the failure class");
12344            assert!(!should_hand_over(Some(&previous), &current));
12345        }
12346        assert_eq!(
12347            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12348            failure_signature("cannot open /var/x.txt: denied")
12349        );
12350        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12351    }
12352
12353    #[test]
12354    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12355        let other = FailClass::Other("x".into());
12356        assert!(should_hand_over(None, &FailClass::Quota));
12357        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12358        assert!(should_hand_over(
12359            Some(&FailClass::Timeout),
12360            &FailClass::Timeout
12361        ));
12362        assert!(should_hand_over(None, &other));
12363        assert!(!should_hand_over(Some(&other), &other));
12364        assert!(should_hand_over(
12365            Some(&other),
12366            &FailClass::Other("y".into())
12367        ));
12368        // A quota or timeout in between ends the run of identical failures.
12369        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12370        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12371    }
12372
12373    #[test]
12374    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12375        let a = SeatState::new("judge-1", "alpha", 7);
12376        let b = handover_seat("judge-1", "beta", 7);
12377        assert_ne!(a.claude_session, b.claude_session);
12378        assert_eq!(b.turns, 0);
12379    }
12380
12381    #[test]
12382    fn a_timeout_is_classified_apart_from_other_failures() {
12383        assert_eq!(
12384            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12385            Some(FailClass::Timeout)
12386        );
12387        assert!(matches!(
12388            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12389            Some(FailClass::Other(_))
12390        ));
12391    }
12392
12393    #[test]
12394    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12395        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12396        let tried = BTreeSet::from(["beta".to_owned()]);
12397        // beta sits at index 1; the next candidate is gamma, never alpha —
12398        // which is very likely a different candidate slot's own agent.
12399        let next = next_untried_in_roster(&roster, 1, &tried);
12400        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12401    }
12402
12403    #[test]
12404    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12405        let roster = vec![spec("alpha"), spec("beta")];
12406        let tried = BTreeSet::from(["beta".to_owned()]);
12407        // beta is the roster's last entry: nothing follows it, and alpha —
12408        // earlier in the roster, almost certainly a different candidate
12409        // slot's own agent — must not be reached by wrapping back to it.
12410        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12411    }
12412
12413    #[test]
12414    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12415        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12416        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12417        // beta (index 1) and gamma (index 2, the only entry after it) have
12418        // both been tried; alpha (index 0) never has, but it comes before
12419        // beta's own position, so there is nothing further for this seat.
12420        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12421    }
12422
12423    #[test]
12424    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12425        let roster = vec![spec("a"), spec("a"), spec("b")];
12426        let tried = BTreeSet::from(["a".to_owned()]);
12427        let next = next_untried_in_roster(&roster, 0, &tried);
12428        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12429    }
12430
12431    #[test]
12432    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12433        let roster = vec![spec("a"), spec("b")];
12434        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12435        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12436    }
12437
12438    #[test]
12439    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12440        let dir = tempfile::tempdir().unwrap();
12441        let bay = dir.path().join("ffff");
12442
12443        // Not there yet: nothing to do, nothing to panic on.
12444        remove_if_empty(&bay);
12445        assert!(!bay.exists());
12446
12447        // Something still inside - the winner's worktree, or a stray file -
12448        // keeps the directory standing.
12449        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12450        remove_if_empty(&bay);
12451        assert!(bay.exists(), "non-empty directory must survive");
12452
12453        // Once the last entry is gone, so is the directory itself.
12454        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12455        remove_if_empty(&bay);
12456        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12457    }
12458
12459    // `round_is_clean` is the exact decision this task fixed: a round with a
12460    // seat that never answered must not read the same as a round every seat
12461    // actually reviewed. These are deterministic and process-free by design —
12462    // the equivalent end-to-end check (a real reviewer timing out under a
12463    // live graph run) is a genuine race against wall-clock contention, and a
12464    // spawn slow enough to blow even a generous budget under a loaded test
12465    // run must not turn this specific regression check flaky.
12466
12467    #[test]
12468    fn a_full_panel_that_found_nothing_is_clean() {
12469        assert!(round_is_clean(
12470            0,
12471            true,
12472            2,
12473            2,
12474            0,
12475            IncompleteReviewPolicy::Block
12476        ));
12477    }
12478
12479    #[test]
12480    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12481        assert!(!round_is_clean(
12482            0,
12483            true,
12484            1,
12485            2,
12486            0,
12487            IncompleteReviewPolicy::Block
12488        ));
12489    }
12490
12491    #[test]
12492    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12493        assert!(!round_is_clean(
12494            1,
12495            true,
12496            1,
12497            2,
12498            0,
12499            IncompleteReviewPolicy::Warn
12500        ));
12501    }
12502
12503    #[test]
12504    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12505        assert!(round_is_clean(
12506            0,
12507            true,
12508            1,
12509            2,
12510            0,
12511            IncompleteReviewPolicy::Warn
12512        ));
12513    }
12514
12515    #[test]
12516    fn a_full_panel_with_an_open_finding_is_not_clean() {
12517        assert!(!round_is_clean(
12518            1,
12519            true,
12520            2,
12521            2,
12522            0,
12523            IncompleteReviewPolicy::Block
12524        ));
12525    }
12526
12527    #[test]
12528    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12529        assert!(!round_is_clean(
12530            0,
12531            false,
12532            2,
12533            2,
12534            0,
12535            IncompleteReviewPolicy::Block
12536        ));
12537    }
12538
12539    // The stall this task closes: under the default `block` policy, a seat
12540    // missing only because it was rate limited must not force a wait for a
12541    // session limit that will not lift by the next round. `round_is_clean`
12542    // is where that quorum carve-out lives; the review loop around it never
12543    // changes what a reviewer's vote or a finding's severity means.
12544
12545    #[test]
12546    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12547        // 1 of 2 answered, and the one missing was quota'd — the exact
12548        // "review-2 rate limited (quota)" shape from the field report.
12549        assert!(round_is_clean(
12550            0,
12551            true,
12552            1,
12553            2,
12554            1,
12555            IncompleteReviewPolicy::Block
12556        ));
12557    }
12558
12559    #[test]
12560    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12561        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12562        // not a quota loss (`quota_missing` stays 0) — worth another try.
12563        assert!(!round_is_clean(
12564            0,
12565            true,
12566            1,
12567            2,
12568            0,
12569            IncompleteReviewPolicy::Block
12570        ));
12571    }
12572
12573    #[test]
12574    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12575        assert!(!round_is_clean(
12576            1,
12577            true,
12578            1,
12579            2,
12580            1,
12581            IncompleteReviewPolicy::Block
12582        ));
12583        assert!(!round_is_clean(
12584            0,
12585            false,
12586            1,
12587            2,
12588            1,
12589            IncompleteReviewPolicy::Block
12590        ));
12591    }
12592
12593    #[test]
12594    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12595        // Every seat quota'd, nobody answered: there is no panel to decide
12596        // on, so this must fall through to the existing block-and-retry
12597        // fallback rather than call an unreviewed patch clean.
12598        assert!(!round_is_clean(
12599            0,
12600            true,
12601            0,
12602            2,
12603            2,
12604            IncompleteReviewPolicy::Block
12605        ));
12606    }
12607
12608    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12609        CommandOutcome {
12610            command: "test".to_owned(),
12611            code,
12612            output_tail: String::new(),
12613            duration_ms: 0,
12614            resource_blocked,
12615        }
12616    }
12617
12618    #[test]
12619    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12620        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12621        assert!(
12622            !verify_inconclusive(&[outcome(Some(1), false)]),
12623            "an ordinary failure is still evidence about the patch"
12624        );
12625        assert!(verify_inconclusive(&[outcome(None, true)]));
12626        assert!(
12627            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12628            "one inconclusive outcome taints the whole batch"
12629        );
12630        assert!(!verify_inconclusive(&[]));
12631    }
12632
12633    #[tokio::test]
12634    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12635        // Alive for the first two checks, then dead - confirms the loop
12636        // actually re-polls rather than deciding once and sleeping out the
12637        // ceiling regardless.
12638        let calls = std::sync::atomic::AtomicUsize::new(0);
12639        let started = Instant::now();
12640        wait_for_pids_with(
12641            &[123],
12642            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12643            Duration::from_millis(5),
12644            Duration::from_secs(5),
12645        )
12646        .await;
12647        assert!(
12648            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12649            "must keep checking rather than deciding on the first answer"
12650        );
12651        assert!(
12652            started.elapsed() < Duration::from_secs(1),
12653            "must return the moment it is confirmed dead, not wait out the ceiling"
12654        );
12655    }
12656
12657    #[tokio::test]
12658    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12659        let started = Instant::now();
12660        wait_for_pids_with(
12661            &[123],
12662            |_| true, // never reports dead
12663            Duration::from_millis(5),
12664            Duration::from_millis(30),
12665        )
12666        .await;
12667        let elapsed = started.elapsed();
12668        assert!(
12669            elapsed >= Duration::from_millis(30),
12670            "must not give up before its own ceiling: {elapsed:?}"
12671        );
12672        assert!(
12673            elapsed < Duration::from_secs(1),
12674            "must not wait past its own ceiling either: {elapsed:?}"
12675        );
12676    }
12677
12678    #[tokio::test]
12679    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12680        let started = Instant::now();
12681        wait_for_pids_with(
12682            &[],
12683            |_| true,
12684            Duration::from_secs(5),
12685            Duration::from_secs(5),
12686        )
12687        .await;
12688        assert!(
12689            started.elapsed() < Duration::from_millis(200),
12690            "an empty pid list has nothing to confirm"
12691        );
12692    }
12693
12694    // `review_conclusion` is the exact decision the review hand-off task
12695    // fixed: a round budget spent (or a tree that stopped moving) must not
12696    // collapse into `Blocked` regardless of what verification actually
12697    // said. Deterministic and process-free for the same reason the
12698    // `round_is_clean` family above is.
12699    fn review_round(
12700        clean: bool,
12701        blocking: usize,
12702        answered: usize,
12703        expected: usize,
12704        progressed: bool,
12705        e2e_ok: bool,
12706    ) -> ReviewRound {
12707        ReviewRound {
12708            round: 1,
12709            head: "h".to_owned(),
12710            verified_head: None,
12711            verified_at: None,
12712            reviews: Vec::new(),
12713            e2e: vec![CommandOutcome {
12714                command: "test".to_owned(),
12715                code: Some(if e2e_ok { 0 } else { 1 }),
12716                output_tail: String::new(),
12717                duration_ms: 0,
12718                resource_blocked: false,
12719            }],
12720            verify_retried: false,
12721            e2e_deferred: false,
12722            e2e_defer_reason: None,
12723            fix: None,
12724            blocking,
12725            answered,
12726            expected,
12727            clean,
12728            progressed,
12729            vote_split: false,
12730            reconsideration: Vec::new(),
12731            verdict: None,
12732        }
12733    }
12734
12735    #[test]
12736    fn review_conclusion_is_none_when_nothing_has_run() {
12737        assert_eq!(review_conclusion(&[], 3), None);
12738    }
12739
12740    #[test]
12741    fn review_conclusion_is_none_while_rounds_remain() {
12742        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12743        assert_eq!(review_conclusion(&rounds, 3), None);
12744    }
12745
12746    #[test]
12747    fn review_conclusion_is_gating_once_a_round_is_clean() {
12748        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12749        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12750    }
12751
12752    #[test]
12753    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12754        let rounds = vec![
12755            review_round(false, 1, 2, 2, true, true),
12756            review_round(false, 1, 2, 2, true, true),
12757        ];
12758        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12759    }
12760
12761    #[test]
12762    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12763        let rounds = vec![
12764            review_round(false, 1, 2, 2, true, true),
12765            review_round(false, 1, 2, 2, true, false),
12766        ];
12767        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12768    }
12769
12770    #[test]
12771    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12772        // Magi never got a command to run against this round's own head — a
12773        // resource-blocked attempt, not a red one — so this must never
12774        // settle on `Blocked` the way a genuine e2e failure would. `None`
12775        // here is what tells `Runner::review_loop` to retry the check
12776        // itself rather than trust this cheap recomputation with a verdict
12777        // it cannot actually produce.
12778        let mut blocked = review_round(false, 1, 2, 2, true, false);
12779        blocked.e2e[0].resource_blocked = true;
12780        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12781        assert_eq!(review_conclusion(&rounds, 2), None);
12782    }
12783
12784    #[test]
12785    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12786        // Missing input, not a verified tree — never a hand-off candidate.
12787        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12788        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12789    }
12790
12791    #[test]
12792    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12793        let rounds = vec![
12794            review_round(false, 1, 2, 2, false, true),
12795            review_round(false, 1, 2, 2, false, true),
12796        ];
12797        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12798    }
12799
12800    fn secs(n: u64) -> Duration {
12801        Duration::from_secs(n)
12802    }
12803
12804    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12805    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12806    #[tokio::test]
12807    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12808        ask_test_home();
12809        let tmp = tempfile::tempdir().unwrap();
12810        let repo = tmp.path().join("repo");
12811        std::fs::create_dir_all(&repo).unwrap();
12812        init_repo(&repo);
12813        let g = |args: &[&str]| {
12814            let out = std::process::Command::new("git")
12815                .args(args)
12816                .current_dir(&repo)
12817                .quiet()
12818                .output()
12819                .unwrap();
12820            assert!(
12821                out.status.success(),
12822                "{}",
12823                String::from_utf8_lossy(&out.stderr)
12824            );
12825        };
12826        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12827        g(&["checkout", "-b", "historical"]);
12828        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12829        g(&["add", "."]);
12830        g(&["commit", "-m", "old alternative"]);
12831        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12832        g(&["checkout", "-b", "magi/task-selected", &base]);
12833        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12834        g(&["add", "."]);
12835        g(&["commit", "-m", "measurement infrastructure"]);
12836        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12837        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12838        g(&["add", "."]);
12839        g(&["commit", "-m", "dependent implementation"]);
12840        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12841        g(&["checkout", "main"]);
12842        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12843        g(&["add", "."]);
12844        g(&["commit", "-m", "unrelated base change"]);
12845        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12846        let origin = tmp.path().join("origin.git");
12847        g(&[
12848            "clone",
12849            "--bare",
12850            repo.to_str().unwrap(),
12851            origin.to_str().unwrap(),
12852        ]);
12853        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12854        g(&["fetch", "origin"]);
12855        let selection = crate::pr_recovery::Selection {
12856            url: "https://example.test/project/pull/1".into(),
12857            branch: "magi/task-selected".into(),
12858            head: head.clone(),
12859            source_head: None,
12860            source_run: None,
12861            source_proof: None,
12862            disposition: Some(crate::pr_recovery::Disposition::Review),
12863            instruction: "preserve measured behavior".into(),
12864            approved_at: jiff::Timestamp::now(),
12865        };
12866        let mut template = runner_at(RunStatus::Reviewing);
12867        template.state.repo = repo.clone();
12868        template.state.base_commit = current.clone();
12869        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12870        template.state.origin = Some(crate::run::Origin::queue("selected"));
12871        template.state.pr_head = Some(selection.branch.clone());
12872        template.state.pr_recovery = Some(selection.clone());
12873        let mut runner =
12874            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12875                .await
12876                .unwrap();
12877        let wt = runner.state.winner().unwrap().worktree.clone();
12878        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12879        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12880        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12881        assert!(runner.state.seeds.is_empty());
12882        assert_eq!(runner.state.instruction, "preserve measured behavior");
12883        runner.sync_to_base().await.unwrap();
12884        assert_eq!(
12885            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12886            "prerequisite"
12887        );
12888        assert_eq!(
12889            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12890            "measured implementation"
12891        );
12892        assert_eq!(
12893            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12894            "current main change"
12895        );
12896        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12897        assert_eq!(
12898            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12899            head
12900        );
12901
12902        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12903        let mut partial = runner_at(RunStatus::Reviewing);
12904        partial.state.repo = repo.clone();
12905        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12906        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12907            run: Some("exit 1".into()),
12908            ..Default::default()
12909        }];
12910        let selection = runner.state.pr_recovery.clone().unwrap();
12911        partial.state.pr_recovery = Some(selection.clone());
12912        partial.state.pr_head = Some(selection.branch.clone());
12913        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12914        let id = partial.state.id.clone();
12915        let error = match Runner::open_selected(
12916            &repo,
12917            partial.state,
12918            partial.roles,
12919            1,
12920            runner.state.base_commit.clone(),
12921            selection,
12922        )
12923        .await
12924        {
12925            Ok(_) => panic!("setup must fail"),
12926            Err(e) => e,
12927        };
12928        assert_eq!(
12929            error
12930                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12931                .unwrap()
12932                .run,
12933            id
12934        );
12935        let failed = RunState::load(&id).unwrap();
12936        assert_eq!(failed.status, RunStatus::Blocked);
12937        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12938        assert!(Runner::resume(&id).is_err());
12939        runner.state.status = RunStatus::Ready;
12940        runner.state.driver_exited = true;
12941        runner.state.save().unwrap();
12942        let mut retry = runner_at(RunStatus::Reviewing);
12943        retry.state.repo = repo.clone();
12944        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12945        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12946        reselected.approved_at = jiff::Timestamp::now();
12947        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12948        retry.state.pr_head = Some(reselected.branch.clone());
12949        retry.state.pr_recovery = Some(reselected.clone());
12950        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12951            .await
12952            .unwrap();
12953        let restarted = Runner::open_selected(
12954            &repo,
12955            retry.state,
12956            retry.roles,
12957            1,
12958            runner.state.base_commit.clone(),
12959            reselected,
12960        )
12961        .await
12962        .unwrap();
12963        assert_eq!(
12964            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12965                .await
12966                .unwrap(),
12967            head
12968        );
12969        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12970        assert_eq!(
12971            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12972            head
12973        );
12974    }
12975
12976    #[tokio::test]
12977    #[cfg(unix)]
12978    async fn fallback_and_interrupted_restoration_require_coordination() {
12979        use std::os::fd::AsRawFd as _;
12980        ask_test_home();
12981        let tmp = tempfile::tempdir().unwrap();
12982        let mut runner = registered_runner(tmp.path()).await;
12983        runner.state.config.graph.timeout_resource_wait = 1;
12984        let winner = runner.state.candidates[0].clone();
12985        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12986        let leftover = winner.worktree.join("leftover");
12987        std::fs::write(&leftover, "untrusted edits").unwrap();
12988        let next = tmp.path().join("next-fixer");
12989        let mut failed = spec("first");
12990        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12991        let mut successor = spec("second");
12992        successor.command = vec![
12993            "sh".into(),
12994            "-c".into(),
12995            format!("touch '{}'; echo done", next.display()),
12996        ];
12997        let attempts = vec![
12998            (failed.clone(), "first".into()),
12999            (successor.clone(), "second".into()),
13000        ];
13001        let lock_path = runner.state.repo.join(".git/updater.lock");
13002        let external = std::fs::File::create(&lock_path).unwrap();
13003        assert_eq!(
13004            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13005            0
13006        );
13007        let artifacts = tmp.path().join("artifacts");
13008        let build = |spec, seat| SeatJob {
13009            spec,
13010            seat,
13011            cwd: winner.worktree.clone(),
13012            prompt: "fixture".into(),
13013            timeout: Duration::from_secs(10),
13014            allow_write: true,
13015            sessions: false,
13016            artifacts: artifacts.clone(),
13017            stem: "fix".into(),
13018            handover: None,
13019        };
13020        let (_, _, out) = runner
13021            .ask_fixer_among(
13022                attempts.clone(),
13023                "graft",
13024                None,
13025                Some((&winner.worktree, &head)),
13026                build,
13027            )
13028            .await;
13029        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13030        assert!(leftover.exists(), "contention must prevent clean");
13031        assert!(!next.exists());
13032        assert!(runner.state.handovers.is_empty());
13033        // Early interrupted-repair rollback uses the same protected route.
13034        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13035            from_run: "earlier".into(),
13036            generation: 1,
13037            mode: crate::recovery::Mode::VerifyRepair,
13038            reasons: Vec::new(),
13039            reviewed_head: None,
13040            rounds: Vec::new(),
13041            open_findings: Vec::new(),
13042            evidence: None,
13043            criteria_inherited: true,
13044            repair: crate::recovery::Repair::Started {
13045                before: head.clone(),
13046            },
13047        });
13048        assert!(
13049            runner
13050                .recovery_repair(&winner)
13051                .await
13052                .unwrap_err()
13053                .is::<crate::repo_lock::ResourceBlocked>()
13054        );
13055        assert!(matches!(
13056            runner.state.recovery.as_ref().unwrap().repair,
13057            crate::recovery::Repair::Started { .. }
13058        ));
13059        assert!(leftover.exists());
13060        drop(external);
13061        runner.recovery_repair(&winner).await.unwrap();
13062        assert!(!leftover.exists());
13063        assert!(matches!(
13064            runner.state.recovery.as_ref().unwrap().repair,
13065            crate::recovery::Repair::Done { .. }
13066        ));
13067        // A hard lock-open failure also leaves HEAD and untracked work intact.
13068        std::fs::remove_file(&lock_path).unwrap();
13069        std::fs::create_dir(&lock_path).unwrap();
13070        std::fs::write(&leftover, "untrusted edits").unwrap();
13071        let (_, _, out) = runner
13072            .ask_fixer_among(
13073                attempts,
13074                "graft",
13075                None,
13076                Some((&winner.worktree, &head)),
13077                build,
13078            )
13079            .await;
13080        assert!(matches!(out, AgentOutcome::Failed(_)));
13081        assert!(leftover.exists());
13082        assert!(!next.exists());
13083        assert_eq!(
13084            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13085            head
13086        );
13087    }
13088
13089    #[cfg(unix)]
13090    async fn registered_runner(root: &Path) -> Runner {
13091        ask_test_home();
13092        let repo = root.join("repo");
13093        std::fs::create_dir_all(&repo).unwrap();
13094        init_repo(&repo);
13095        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13096        let mut runner = runner_at(RunStatus::Prep);
13097        runner.state.repo = repo.clone();
13098        runner.state.config.coordination = crate::config::test_coordination();
13099        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13100        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13101        runner.state.config.graph.advise = false;
13102        runner.roles.implementers = vec![spec("implement")];
13103        runner.roles.judges = vec![spec("judge")];
13104        runner.prep().await.unwrap();
13105        runner
13106    }
13107
13108    #[tokio::test]
13109    #[cfg(unix)]
13110    async fn parked_implementer_persists_completed_work() {
13111        for registered in [false, true] {
13112            let tmp = tempfile::tempdir().unwrap();
13113            let mut runner = registered_runner(tmp.path()).await;
13114            if !registered {
13115                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13116            }
13117            let marker = tmp.path().join("started");
13118            let release = tmp.path().join("release");
13119            let mut implementer = spec("implement");
13120            implementer.command = vec![
13121                "sh".into(),
13122                "-c".into(),
13123                format!(
13124                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13125                    marker.display(),
13126                    release.display()
13127                ),
13128            ];
13129            runner.roles.implementers = vec![implementer.clone()];
13130            runner.state.config.agents = vec![implementer];
13131            let pause = runner.pause.clone();
13132            let (result, ()) = tokio::join!(runner.implement(), async {
13133                tokio::time::timeout(Duration::from_secs(10), async {
13134                    while !marker.exists() {
13135                        tokio::time::sleep(Duration::from_millis(10)).await;
13136                    }
13137                })
13138                .await
13139                .unwrap();
13140                pause.park();
13141                std::fs::write(&release, "go").unwrap();
13142            });
13143            result.unwrap();
13144            let candidate = &runner.state.candidates[0];
13145            assert!(candidate.commits > 0, "{candidate:?}");
13146            assert!(candidate.summary.contains("saved-result"));
13147            assert_eq!(candidate.files, 1);
13148            let before = candidate.commits;
13149            runner.implement().await.unwrap();
13150            assert_eq!(runner.state.candidates[0].commits, before);
13151        }
13152    }
13153
13154    #[tokio::test]
13155    #[cfg(unix)]
13156    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13157        let tmp = tempfile::tempdir().unwrap();
13158        let mut runner = registered_runner(tmp.path()).await;
13159        let winner = runner.state.candidates[0].clone();
13160        let cache = tmp.path().join("cache");
13161        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13162        runner.state.config.graph.timeout_resource_wait = 0;
13163        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13164        let crate::cache::AcquireOutcome::Acquired(held) =
13165            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13166        else {
13167            panic!("cache must be free")
13168        };
13169        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13170            from_run: "earlier".into(),
13171            generation: 1,
13172            mode: crate::recovery::Mode::VerifyRepair,
13173            reasons: Vec::new(),
13174            reviewed_head: None,
13175            rounds: Vec::new(),
13176            open_findings: Vec::new(),
13177            evidence: None,
13178            criteria_inherited: true,
13179            repair: crate::recovery::Repair::NotStarted,
13180        });
13181        runner.recovery_repair(&winner).await.unwrap();
13182        assert!(matches!(
13183            runner.state.recovery.as_ref().unwrap().repair,
13184            crate::recovery::Repair::WaitingForRestore { .. }
13185        ));
13186        drop(held);
13187        runner.state.resource_block = None;
13188        runner.state.config.graph.timeout_resource_wait = 1;
13189        runner.recovery_repair(&winner).await.unwrap();
13190        assert!(matches!(
13191            runner.state.recovery.as_ref().unwrap().repair,
13192            crate::recovery::Repair::Done { .. }
13193        ));
13194        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13195    }
13196
13197    #[tokio::test]
13198    #[cfg(unix)]
13199    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13200        use std::os::fd::AsRawFd as _;
13201        for graft in [false, true] {
13202            let tmp = tempfile::tempdir().unwrap();
13203            let mut runner = registered_runner(tmp.path()).await;
13204            runner.state.config.graph.timeout_resource_wait = 1;
13205            let winner = runner.state.candidates[0].clone();
13206            let marker = tmp.path().join("fixer-started");
13207            let mut fixer = spec("fixer");
13208            fixer.command = vec![
13209                "sh".into(),
13210                "-c".into(),
13211                format!("echo started >> '{}'; echo done", marker.display()),
13212            ];
13213            runner.roles.fixer = Some(vec![fixer]);
13214            if graft {
13215                let mut loser = winner.clone();
13216                loser.label = 'B';
13217                loser.index = 1;
13218                runner.state.candidates.push(loser);
13219                runner.state.config.graph.graft_runner_up = true;
13220                runner.state.tally = Some(Tally {
13221                    first_choice: BTreeMap::new(),
13222                    borda: BTreeMap::new(),
13223                    winner: winner.label,
13224                    rankings: 1,
13225                    unanimous_initial: true,
13226                    deliberated: false,
13227                    changed_votes: 0,
13228                    unanimous_final: true,
13229                    tie_break: None,
13230                    judges: 1,
13231                    present: 1,
13232                    quorum: 1,
13233                    met_quorum: true,
13234                    uncontested: None,
13235                });
13236                runner.state.judgements.push(crate::run::Judgement {
13237                    judge: 1,
13238                    seat: "judge-1".into(),
13239                    agent: "judge".into(),
13240                    ranking: vec![winner.label, 'B'],
13241                    reasons: BTreeMap::new(),
13242                    confidence: None,
13243                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13244                    order: vec![0, 1],
13245                    failed: None,
13246                    duration_ms: 0,
13247                });
13248            } else {
13249                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13250                    from_run: "earlier".into(),
13251                    generation: 1,
13252                    mode: crate::recovery::Mode::VerifyRepair,
13253                    reasons: Vec::new(),
13254                    reviewed_head: None,
13255                    rounds: Vec::new(),
13256                    open_findings: Vec::new(),
13257                    evidence: None,
13258                    criteria_inherited: true,
13259                    repair: crate::recovery::Repair::NotStarted,
13260                });
13261            }
13262            let reader = crate::repo_lock::SeatLease::for_checkout(
13263                &winner.worktree,
13264                false,
13265                crate::config::test_coordination().as_ref(),
13266            )
13267            .await
13268            .unwrap();
13269            let pause = runner.pause.clone();
13270            let (result, ()) = tokio::join!(
13271                async {
13272                    if graft {
13273                        runner.graft().await
13274                    } else {
13275                        runner.recovery_repair(&winner).await
13276                    }
13277                },
13278                async {
13279                    tokio::time::sleep(Duration::from_millis(100)).await;
13280                    pause.park();
13281                }
13282            );
13283            result.unwrap();
13284            assert!(!marker.exists(), "a blocked seat must not spawn");
13285            // Exercise the persisted state, as a binary replacement/resume does.
13286            runner.state = RunState::load(&runner.state.id).unwrap();
13287            if graft {
13288                assert_eq!(
13289                    runner.state.graft.as_ref().unwrap().outcome,
13290                    GraftOutcome::WaitingForRestore
13291                );
13292            } else {
13293                assert!(matches!(
13294                    runner.state.recovery.as_ref().unwrap().repair,
13295                    crate::recovery::Repair::WaitingForRestore { .. }
13296                ));
13297            }
13298            drop(reader);
13299            runner.pause = Pause::new();
13300            runner.state.resource_block = None;
13301            // Deferred restoration must still respect an external lock owner.
13302            let external =
13303                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13304            assert_eq!(
13305                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13306                0
13307            );
13308            let leftover = winner.worktree.join("leftover");
13309            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13310            let result = if graft {
13311                runner.graft().await
13312            } else {
13313                runner.recovery_repair(&winner).await
13314            };
13315            assert!(
13316                result
13317                    .unwrap_err()
13318                    .is::<crate::repo_lock::ResourceBlocked>()
13319            );
13320            assert!(leftover.exists());
13321            assert!(!marker.exists());
13322            drop(external);
13323            runner.state.resource_block = None;
13324            let result = if graft {
13325                runner.graft().await
13326            } else {
13327                runner.recovery_repair(&winner).await
13328            };
13329            result.unwrap();
13330            assert!(!leftover.exists(), "restore must precede retry");
13331            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13332            if graft {
13333                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13334            } else {
13335                assert!(matches!(
13336                    runner.state.recovery.as_ref().unwrap().repair,
13337                    crate::recovery::Repair::Done { .. }
13338                ));
13339            }
13340        }
13341    }
13342
13343    #[tokio::test]
13344    #[cfg(unix)]
13345    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13346        use std::os::fd::AsRawFd as _;
13347        let tmp = tempfile::tempdir().unwrap();
13348        let mut runner = registered_runner(tmp.path()).await;
13349        runner.state.config.graph.timeout_resource_wait = 1;
13350        let winner = runner.state.candidates[0].clone();
13351        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13352        runner.state.tally = Some(Tally {
13353            first_choice: BTreeMap::new(),
13354            borda: BTreeMap::new(),
13355            winner: winner.label,
13356            rankings: 0,
13357            unanimous_initial: false,
13358            deliberated: false,
13359            changed_votes: 0,
13360            unanimous_final: false,
13361            tie_break: None,
13362            judges: 0,
13363            present: 0,
13364            quorum: 0,
13365            met_quorum: true,
13366            uncontested: Some("fixture".into()),
13367        });
13368        runner.state.graft = Some(GraftRecord {
13369            pre: pre.clone(),
13370            post: None,
13371            lines: None,
13372            limit: None,
13373            agent: None,
13374            outcome: GraftOutcome::Running,
13375            requested: Vec::new(),
13376            adoption: None,
13377            adoption_error: None,
13378        });
13379        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13380        git::commit_all(&winner.worktree, "unfinished graft")
13381            .await
13382            .unwrap();
13383        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13384        let leftover = winner.worktree.join("leftover");
13385        std::fs::write(&leftover, "untrusted edits").unwrap();
13386        let path = runner.state.repo.join(".git/updater.lock");
13387        let external = std::fs::File::create(&path).unwrap();
13388        assert_eq!(
13389            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13390            0
13391        );
13392        assert!(
13393            runner
13394                .graft()
13395                .await
13396                .unwrap_err()
13397                .is::<crate::repo_lock::ResourceBlocked>()
13398        );
13399        assert!(matches!(
13400            runner.state.graft.as_ref().unwrap().outcome,
13401            GraftOutcome::Running
13402        ));
13403        assert_eq!(
13404            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13405            unfinished
13406        );
13407        assert!(leftover.exists());
13408        drop(external);
13409        std::fs::remove_file(&path).unwrap();
13410        std::fs::create_dir(&path).unwrap();
13411        assert!(runner.graft().await.is_err());
13412        assert!(matches!(
13413            runner.state.graft.as_ref().unwrap().outcome,
13414            GraftOutcome::Running
13415        ));
13416        assert_eq!(
13417            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13418            unfinished
13419        );
13420        assert!(leftover.exists());
13421        std::fs::remove_dir(&path).unwrap();
13422        runner.graft().await.unwrap();
13423        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13424        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13425        assert!(!leftover.exists());
13426    }
13427
13428    #[tokio::test]
13429    #[cfg(unix)]
13430    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13431        use std::os::fd::AsRawFd as _;
13432        let tmp = tempfile::tempdir().unwrap();
13433        let mut runner = registered_runner(tmp.path()).await;
13434        runner.state.prep_complete = Some(false);
13435        runner.state.config.graph.timeout_resource_wait = 1;
13436        runner.state.save().unwrap();
13437        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13438        assert_eq!(
13439            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13440            0
13441        );
13442        runner.execute().await.unwrap();
13443        let disk = RunState::load(&runner.state.id).unwrap();
13444        assert!(disk.driver_exited);
13445        assert_eq!(disk.driver_pid, None);
13446        assert_eq!(runner.state.status, RunStatus::Blocked);
13447        assert_eq!(
13448            runner.state.resource_block.as_ref().unwrap().kind,
13449            "repository coordination"
13450        );
13451        assert!(runner.state.prep_failure.is_none());
13452        assert!(runner.state.seats.is_empty());
13453        runner.pause.park();
13454        // Include headroom for Git subprocesses under parallel test load;
13455        // the repository contention deadline itself remains one second.
13456        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13457            .await
13458            .unwrap()
13459            .unwrap();
13460        assert!(runner.state.parked);
13461        assert!(runner.state.prep_failure.is_none());
13462        let disk = RunState::load(&runner.state.id).unwrap();
13463        assert!(disk.driver_exited);
13464        assert_eq!(disk.driver_pid, None);
13465    }
13466
13467    #[tokio::test]
13468    #[cfg(unix)]
13469    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13470        use std::os::fd::AsRawFd as _;
13471        let tmp = tempfile::tempdir().unwrap();
13472        let mut runner = registered_runner(tmp.path()).await;
13473        runner.state.prep_complete = Some(false);
13474        runner.state.config.graph.timeout_resource_wait = 30;
13475        runner.state.save().unwrap();
13476        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13477        assert_eq!(
13478            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13479            0
13480        );
13481        assert!(
13482            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13483                .await
13484                .is_err()
13485        );
13486        let disk = RunState::load(&runner.state.id).unwrap();
13487        assert!(disk.driver_exited);
13488        assert_eq!(disk.driver_pid, None);
13489        assert!(disk.status.resumable());
13490        assert!(runner.state.driver_exited);
13491        assert_eq!(runner.state.driver_pid, None);
13492        runner.state.save().unwrap();
13493        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13494    }
13495
13496    #[tokio::test]
13497    #[cfg(unix)]
13498    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13499        let tmp = tempfile::tempdir().unwrap();
13500        let mut runner = registered_runner(tmp.path()).await;
13501        let winner = runner.state.candidates[0].clone();
13502        let cache = tmp.path().join("cache");
13503        let started = tmp.path().join("pre-gate-started");
13504        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13505        runner.state.config.verify.gate =
13506            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13507        let home = crate::run::home();
13508        let owner =
13509            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13510        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13511            crate::cache::AcquireOutcome::Acquired(g) => g,
13512            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13513        };
13514        let mut gate = Box::pin(runner.run_gate(&winner));
13515        assert!(
13516            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13517                .await
13518                .is_err()
13519        );
13520        assert!(!started.exists(), "pre-gate must wait for the cache too");
13521        // Simulate the wave that already owns the cache entering invoke.
13522        // It must obtain the editor reservation and finish without waiting
13523        // for the gate that is queued behind its cache.
13524        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13525        let editor = crate::repo_lock::SeatLease::checkout_with(
13526            &winner.worktree,
13527            true,
13528            crate::config::test_coordination().as_ref(),
13529            &wait,
13530        )
13531        .await
13532        .unwrap();
13533        drop(editor);
13534        guard.release();
13535        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13536            .await
13537            .unwrap()
13538            .unwrap();
13539        drop(gate);
13540        assert!(outcomes.iter().all(CommandOutcome::ok));
13541        assert!(started.exists());
13542    }
13543
13544    #[tokio::test]
13545    #[cfg(unix)]
13546    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13547        use std::os::fd::AsRawFd as _;
13548
13549        ask_test_home();
13550        let tmp = tempfile::tempdir().unwrap();
13551        let repo = tmp.path().join("repo");
13552        std::fs::create_dir_all(&repo).unwrap();
13553        init_repo(&repo);
13554        let common = repo.join(".git");
13555        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13556        let mut runner = runner_at(RunStatus::Prep);
13557        runner.state.repo = repo.clone();
13558        runner.state.config.coordination = crate::config::test_coordination();
13559        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13560        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13561        runner.state.config.graph.advise = false;
13562        runner.roles.implementers = vec![spec("implement")];
13563        runner.roles.judges = vec![spec("judge")];
13564        runner.prep().await.unwrap();
13565        let winner = runner.state.candidates[0].clone();
13566        // Emulate edits already left by a completed seat. The fixture CLI
13567        // only reports completion; it performs no repository operations.
13568        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13569        let completed = tmp.path().join("seat-completed");
13570        runner.roles.implementers[0].command = vec![
13571            "sh".into(),
13572            "-c".into(),
13573            format!("touch '{}'; echo completed", completed.display()),
13574        ];
13575        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13576        assert_eq!(
13577            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13578            0
13579        );
13580        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13581        let mut implement = Box::pin(runner.implement());
13582        tokio::select! {
13583            result = &mut implement => panic!("rescue must wait: {result:?}"),
13584            result = tokio::time::timeout(Duration::from_secs(10), async {
13585                while !completed.exists() {
13586                    tokio::time::sleep(Duration::from_millis(20)).await;
13587                }
13588            }) => result.unwrap(),
13589        }
13590        assert!(
13591            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13592                .await
13593                .is_err()
13594        );
13595        assert_eq!(
13596            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13597            before
13598        );
13599        assert_eq!(
13600            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13601            0
13602        );
13603        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13604            .await
13605            .unwrap()
13606            .unwrap();
13607        drop(implement);
13608        assert_ne!(
13609            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13610            before
13611        );
13612        assert!(git::is_clean(&winner.worktree).await.unwrap());
13613
13614        let started = tmp.path().join("pre-gate-started");
13615        let proceed = tmp.path().join("pre-gate-proceed");
13616        let checked = tmp.path().join("gate-ran");
13617        runner.state.config.verify.pre_gate = vec![format!(
13618            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13619            started.display(),
13620            proceed.display()
13621        )];
13622        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13623        assert_eq!(
13624            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13625            0
13626        );
13627        let mut gate = Box::pin(runner.run_gate(&winner));
13628        assert!(
13629            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13630                .await
13631                .is_err()
13632        );
13633        assert!(!started.exists());
13634        assert!(!checked.exists());
13635        assert!(git::is_clean(&winner.worktree).await.unwrap());
13636        assert_eq!(
13637            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13638            0
13639        );
13640        tokio::select! {
13641            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13642            result = tokio::time::timeout(Duration::from_secs(10), async {
13643                while !started.exists() {
13644                    tokio::time::sleep(Duration::from_millis(20)).await;
13645                }
13646            }) => result.unwrap(),
13647        }
13648        assert_eq!(
13649            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13650            -1,
13651            "mechanical edits must hold the real lock"
13652        );
13653        std::fs::write(proceed, "ready").unwrap();
13654        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13655            .await
13656            .unwrap()
13657            .unwrap();
13658        drop(gate);
13659        assert!(outcomes.iter().all(CommandOutcome::ok));
13660        assert!(checked.exists());
13661        assert!(runner.state.pre_gate_commit.is_some());
13662        assert!(git::is_clean(&winner.worktree).await.unwrap());
13663        assert_eq!(
13664            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13665            0,
13666            "gate completion must release coordination"
13667        );
13668        drop(outsider);
13669        std::fs::remove_file(common.join("updater.lock")).unwrap();
13670        std::fs::create_dir(common.join("updater.lock")).unwrap();
13671        std::fs::remove_file(&started).unwrap();
13672        std::fs::remove_file(&checked).unwrap();
13673        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13674        let error = runner.run_gate(&winner).await.unwrap_err();
13675        assert!(
13676            error
13677                .to_string()
13678                .contains("acquire mandatory repository lock")
13679        );
13680        assert!(
13681            !started.exists(),
13682            "failed acquisition must prevent pre-gate"
13683        );
13684        assert!(!checked.exists(), "failed acquisition must prevent gate");
13685        assert!(git::is_clean(&winner.worktree).await.unwrap());
13686        assert_eq!(
13687            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13688            head
13689        );
13690    }
13691
13692    #[tokio::test]
13693    #[cfg(unix)]
13694    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13695        use std::os::fd::AsRawFd as _;
13696        use std::time::Duration;
13697
13698        ask_test_home();
13699        let tmp = tempfile::tempdir().unwrap();
13700        let repo = tmp.path().join("repo");
13701        std::fs::create_dir_all(&repo).unwrap();
13702        init_repo(&repo);
13703        let common = repo.join(".git");
13704        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13705        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13706        assert_eq!(
13707            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13708            0
13709        );
13710        let mut runner = runner_at(RunStatus::Prep);
13711        runner.state.repo = repo.clone();
13712        runner.state.config.coordination = crate::config::test_coordination();
13713        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13714        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13715        runner.state.config.graph.advise = true;
13716        runner.state.config.graph.advisors = 1;
13717        runner.roles.implementers = vec![spec("implement")];
13718        runner.roles.judges = vec![spec("judge")];
13719        let root = runner.state.worktree_root();
13720        let started = tmp.path().join("setup-started");
13721        let proceed = tmp.path().join("setup-proceed");
13722        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13723            run: Some(format!(
13724                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13725                started.display(),
13726                proceed.display()
13727            )),
13728            ..Default::default()
13729        }];
13730        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13731        let config = git::git(&repo, &["config", "--local", "--list"])
13732            .await
13733            .unwrap();
13734        let mut prep = Box::pin(runner.prep());
13735        assert!(
13736            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13737                .await
13738                .is_err()
13739        );
13740        assert!(
13741            !root.exists(),
13742            "contention must prevent even checkout creation"
13743        );
13744        assert!(!started.exists(), "contention must prevent setup");
13745        assert_eq!(
13746            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13747            branches
13748        );
13749        assert_eq!(
13750            git::git(&repo, &["config", "--local", "--list"])
13751                .await
13752                .unwrap(),
13753            config
13754        );
13755        drop(lock);
13756        tokio::select! {
13757            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13758            result = tokio::time::timeout(Duration::from_secs(10), async {
13759                while !started.exists() {
13760                    tokio::time::sleep(Duration::from_millis(20)).await;
13761                }
13762            }) => result.unwrap(),
13763        }
13764        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13765        assert_eq!(
13766            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13767            -1
13768        );
13769        assert!(
13770            tokio::time::timeout(
13771                Duration::from_millis(100),
13772                crate::repo_lock::SeatLease::for_checkout(
13773                    &repo,
13774                    true,
13775                    crate::config::test_coordination().as_ref()
13776                )
13777            )
13778            .await
13779            .is_err(),
13780            "preparation must also exclude local editors"
13781        );
13782        std::fs::write(proceed, "ready").unwrap();
13783        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13784            .await
13785            .unwrap()
13786            .unwrap();
13787        drop(prep);
13788        assert_eq!(runner.state.prep_complete, Some(true));
13789        assert!(root.join("judge-1").exists());
13790        assert!(root.join("advisor-1").exists());
13791        assert_eq!(
13792            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13793            0
13794        );
13795    }
13796
13797    #[tokio::test]
13798    #[cfg(unix)]
13799    async fn preparation_lock_failure_prevents_repository_changes() {
13800        ask_test_home();
13801        let tmp = tempfile::tempdir().unwrap();
13802        let repo = tmp.path().join("repo");
13803        std::fs::create_dir_all(&repo).unwrap();
13804        init_repo(&repo);
13805        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13806        // A directory at the required lock path makes opening the real lock
13807        // fail, rather than merely exercising an invalid registration.
13808        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13809        let mut runner = runner_at(RunStatus::Prep);
13810        runner.state.repo = repo.clone();
13811        runner.state.config.coordination = crate::config::test_coordination();
13812        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13813        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13814        runner.roles.implementers = vec![spec("implement")];
13815        let error = runner.prep().await.unwrap_err();
13816        assert!(
13817            error
13818                .to_string()
13819                .contains("acquire mandatory repository lock")
13820        );
13821        assert!(!runner.state.worktree_root().exists());
13822        assert!(runner.state.candidates.is_empty());
13823        assert!(!repo.join(".git/config.worktree").exists());
13824        assert_eq!(
13825            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13826            "* main"
13827        );
13828    }
13829
13830    #[tokio::test]
13831    async fn preparation_records_creation_intent_before_git_can_fail() {
13832        ask_test_home();
13833        let tmp = tempfile::tempdir().unwrap();
13834        let repo = tmp.path().join("repo");
13835        std::fs::create_dir_all(&repo).unwrap();
13836        init_repo(&repo);
13837        let mut runner = runner_at(RunStatus::Prep);
13838        runner.state.repo = repo.clone();
13839        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13840        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13841        runner.state.config.blind.commit_msg_hook = false;
13842        runner.roles.implementers = vec![spec("one")];
13843        let label = blind::assign_labels(1, runner.state.seed)[0];
13844        let branch = runner.state.branch_for(label);
13845        // A conflicting branch makes Git refuse creation before a candidate
13846        // can be registered. The durable intent must still identify it.
13847        git::git(&repo, &["branch", &branch]).await.unwrap();
13848        assert!(runner.prep().await.is_err());
13849        let saved = RunState::load(&runner.state.id).unwrap();
13850        assert!(saved.candidates.is_empty());
13851        let event = saved
13852            .events
13853            .iter()
13854            .find(|e| e.node == "prep-intent")
13855            .unwrap();
13856        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13857        assert_eq!(intent["branch"], branch);
13858        assert_eq!(intent["start"], runner.state.base_commit);
13859        assert_eq!(
13860            intent["worktree"],
13861            runner
13862                .state
13863                .worktree_root()
13864                .join(format!("cand-{label}"))
13865                .to_string_lossy()
13866                .as_ref()
13867        );
13868        assert_eq!(intent["picks"], serde_json::json!([]));
13869    }
13870
13871    #[tokio::test]
13872    async fn invalid_pinned_sources_create_no_partial_worktrees() {
13873        ask_test_home();
13874        let tmp = tempfile::tempdir().unwrap();
13875        let repo = tmp.path().join("repo");
13876        std::fs::create_dir_all(&repo).unwrap();
13877        init_repo(&repo);
13878        let mut runner = runner_at(RunStatus::Prep);
13879        runner.state.repo = repo.clone();
13880        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13881        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13882        runner.state.sources.pick_commits.push(refs::Input {
13883            token: "missing".into(),
13884            sha: "0".repeat(40),
13885            provenance: refs::Provenance::ExplicitCli,
13886        });
13887        assert!(runner.prep().await.is_err());
13888        assert!(runner.state.candidates.is_empty());
13889        assert!(!runner.state.worktree_root().exists());
13890    }
13891
13892    #[tokio::test]
13893    async fn prose_and_legacy_inference_never_import_into_candidates() {
13894        ask_test_home();
13895        let tmp = tempfile::tempdir().unwrap();
13896        let repo = tmp.path().join("repo");
13897        std::fs::create_dir_all(&repo).unwrap();
13898        init_repo(&repo);
13899        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13900        git::git(&repo, &["checkout", "-b", "magi/history/A"])
13901            .await
13902            .unwrap();
13903        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13904        git::git(&repo, &["add", "."]).await.unwrap();
13905        git::git(&repo, &["commit", "-m", "historical implementation"])
13906            .await
13907            .unwrap();
13908        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13909        git::git(&repo, &["checkout", "main"]).await.unwrap();
13910        for instruction in [
13911            format!("Example: magi/history/A at {source}"),
13912            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13913            format!("Never import magi/history/A or {source}"),
13914            format!("Conversation answer: I mentioned {source} only as background"),
13915        ] {
13916            let mut runner = runner_at(RunStatus::Prep);
13917            runner.state.repo = repo.clone();
13918            runner.state.base_commit = base.clone();
13919            runner.state.instruction = instruction.clone();
13920            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13921            runner.state.config.blind.commit_msg_hook = false;
13922            runner.state.seeds = vec![refs::Seed {
13923                token: "magi/history/A".into(),
13924                sha: source.clone(),
13925                kind: refs::SeedKind::Unmerged,
13926                branch: true,
13927                detail: String::new(),
13928            }];
13929            runner.roles.implementers = vec![spec("one")];
13930            assert_eq!(seeded_instruction(&runner.state), instruction);
13931            // Model an old record with inferred seeds but no structured input field.
13932            let mut legacy = serde_json::to_value(&runner.state).unwrap();
13933            legacy.as_object_mut().unwrap().remove("sources");
13934            runner.state = serde_json::from_value(legacy).unwrap();
13935            assert_eq!(runner.state.sources, refs::Sources::default());
13936            runner.state.save().unwrap();
13937            runner.state = RunState::load(&runner.state.id).unwrap();
13938            runner.prep().await.unwrap();
13939            for candidate in &runner.state.candidates {
13940                assert_eq!(
13941                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13942                    base
13943                );
13944                assert!(!candidate.worktree.join("unwanted").exists());
13945            }
13946        }
13947    }
13948
13949    #[tokio::test]
13950    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13951        ask_test_home();
13952        let tmp = tempfile::tempdir().unwrap();
13953        let repo = tmp.path().join("repo");
13954        std::fs::create_dir_all(&repo).unwrap();
13955        init_repo(&repo);
13956        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13957        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13958        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13959        git::git(&repo, &["add", "."]).await.unwrap();
13960        git::git(&repo, &["commit", "-m", "required seed"])
13961            .await
13962            .unwrap();
13963        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13964        git::git(&repo, &["checkout", "main"]).await.unwrap();
13965        let mut runner = runner_at(RunStatus::Prep);
13966        runner.state.repo = repo.clone();
13967        runner.state.base_commit = base.clone();
13968        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13969        runner.state.config.graph.advise = false;
13970        runner.state.config.blind.commit_msg_hook = false;
13971        runner.state.sources =
13972            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13973                .await
13974                .unwrap();
13975        runner.roles.implementers = vec![spec("one"), spec("two")];
13976        runner.roles.judges = vec![spec("judge")];
13977        let label = blind::assign_labels(2, runner.state.seed)[0];
13978        let branch = runner.state.branch_for(label);
13979        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13980        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13981            .await
13982            .unwrap();
13983        runner.state.candidates.push(Candidate {
13984            index: 0,
13985            label,
13986            agent: "one".into(),
13987            branch,
13988            worktree,
13989            summary: String::new(),
13990            stat: String::new(),
13991            files: 0,
13992            commits: 0,
13993            empty: false,
13994            failed: None,
13995            verified_noop: None,
13996            duration_ms: 0,
13997            folded: false,
13998        });
13999        runner.state.save().unwrap();
14000        // Reload the checkpoint immediately before its first cherry-pick.
14001        runner.state = RunState::load(&runner.state.id).unwrap();
14002        assert_eq!(runner.state.prep_complete, Some(false));
14003        runner.prep().await.unwrap();
14004        assert_eq!(runner.state.prep_complete, Some(true));
14005        assert_eq!(runner.state.candidates.len(), 2);
14006        for candidate in &runner.state.candidates {
14007            assert_eq!(
14008                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14009                git::tree_of(&repo, &seed).await.unwrap()
14010            );
14011            assert_eq!(
14012                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14013                "required commit"
14014            );
14015        }
14016        assert!(runner.state.worktree_root().join("judge-1").exists());
14017        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14018        runner.prep().await.unwrap();
14019        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14020
14021        runner.state.prep_complete = Some(false);
14022        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14023            run: Some(format!(
14024                "test -f '{}'",
14025                tmp.path().join("dependency-ready").display()
14026            )),
14027            ..Default::default()
14028        }];
14029        assert!(runner.prep().await.is_err());
14030        assert!(runner.state.prep_failure.is_some());
14031        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14032        runner.state = RunState::load(&runner.state.id).unwrap();
14033        assert!(
14034            runner.prep().await.is_err(),
14035            "correcting the cause alone must not auto-retry"
14036        );
14037        let failed_at = runner.state.events.last().unwrap().at;
14038        assert!(
14039            runner.retry_preparation(failed_at).is_err(),
14040            "old approvals cannot unlock the hold"
14041        );
14042        let mut task = crate::queue::Task::new(
14043            "retry".into(),
14044            "retry setup".into(),
14045            repo.clone(),
14046            crate::queue::Source::Human,
14047        );
14048        task.start(runner.state.id.clone());
14049        task.hold_machine(Some("setup failed".into()));
14050        task.release_operator("cli", None);
14051        runner
14052            .retry_preparation(task.resumption.as_ref().unwrap().at)
14053            .unwrap();
14054        runner.prep().await.unwrap();
14055        assert!(runner.state.prep_failure.is_none());
14056        assert_eq!(runner.state.prep_complete, Some(true));
14057        assert_eq!(runner.state.candidates.len(), 2);
14058        assert!(runner.state.worktree_root().join("judge-1").exists());
14059        for candidate in &runner.state.candidates {
14060            assert_eq!(
14061                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14062                git::tree_of(&repo, &seed).await.unwrap()
14063            );
14064        }
14065    }
14066
14067    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14068        let mut heads = Vec::new();
14069        for candidate in candidates {
14070            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14071        }
14072        heads
14073    }
14074
14075    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14076        ask_test_home();
14077        let tmp = tempfile::tempdir().unwrap();
14078        let repo = tmp.path().join("repo");
14079        std::fs::create_dir(&repo).unwrap();
14080        init_repo(&repo);
14081        let mut runner = runner_at(RunStatus::Reviewing);
14082        runner.state.repo = repo.clone();
14083        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14084        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14085        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14086            let branch = format!("candidate-{label}");
14087            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14088            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14089                .await
14090                .unwrap();
14091            if label == 'B' {
14092                std::fs::write(worktree.join("change"), "winner").unwrap();
14093                git::rescue_commit(&worktree, "candidate change")
14094                    .await
14095                    .unwrap();
14096            }
14097            runner.state.candidates.push(Candidate {
14098                index,
14099                label,
14100                agent: "alpha".into(),
14101                branch,
14102                worktree,
14103                summary: "completed reply".into(),
14104                stat: String::new(),
14105                files: usize::from(label == 'B'),
14106                commits: usize::from(label == 'B'),
14107                empty: label == 'A',
14108                failed: None,
14109                verified_noop: None,
14110                duration_ms: 1,
14111                folded: false,
14112            });
14113        }
14114        runner.state.judge_skipped = true;
14115        runner.state.tally = Some(Tally {
14116            first_choice: BTreeMap::new(),
14117            borda: BTreeMap::new(),
14118            winner: 'B',
14119            rankings: 0,
14120            unanimous_initial: false,
14121            deliberated: false,
14122            changed_votes: 0,
14123            unanimous_final: false,
14124            tie_break: None,
14125            judges: 0,
14126            present: 0,
14127            quorum: 0,
14128            met_quorum: true,
14129            uncontested: Some("only B produced a change".into()),
14130        });
14131        runner.state.save().unwrap();
14132        (tmp, runner)
14133    }
14134
14135    #[tokio::test]
14136    #[cfg(unix)]
14137    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14138        for (solo, failed_checkpoint) in
14139            [(true, false), (false, false), (true, true), (false, true)]
14140        {
14141            let (_tmp, mut runner) = preservation_resume_fixture().await;
14142            if solo {
14143                runner.state.candidates.retain(|c| c.label == 'A');
14144            }
14145            runner.state.candidates[0].empty = false;
14146            runner.state.candidates[0].summary.clear();
14147            runner.state.tally = None;
14148            runner.state.judge_skipped = false;
14149            runner.state.status = RunStatus::Implementing;
14150            runner.state.config.disk.min_free_bytes = u64::MAX;
14151            let mut implementer = spec("alpha");
14152            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14153            runner.roles.implementers = vec![implementer];
14154            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14155            if failed_checkpoint {
14156                std::fs::create_dir_all(&patch).unwrap();
14157            }
14158            runner.state.save().unwrap();
14159            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14160            runner.state = RunState::load(&runner.state.id).unwrap();
14161            assert_eq!(runner.state.status, RunStatus::Blocked);
14162            assert!(runner.state.interrupted_implementations.contains(&'A'));
14163            assert_eq!(runner.state.candidates[0].commits, 0);
14164            assert!(!runner.state.candidates[0].empty);
14165            assert!(runner.state.candidates[0].failed.is_none());
14166            if failed_checkpoint {
14167                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14168                std::fs::remove_dir(patch).unwrap();
14169            }
14170            runner.state.resource_block = None;
14171            runner.state.config.disk.min_free_bytes = 0;
14172            runner.roles.implementers[0].command = vec![
14173                "sh".into(),
14174                "-c".into(),
14175                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14176            ];
14177            runner.implement().await.unwrap();
14178            assert!(runner.state.interrupted_implementations.is_empty());
14179            assert!(runner.state.candidate_preservation.is_empty());
14180            assert_eq!(runner.state.candidates[0].commits, 1);
14181            assert!(!runner.state.candidates[0].empty);
14182            assert!(
14183                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14184                    .unwrap()
14185                    .contains("retried empty interrupted seat")
14186            );
14187            if !solo {
14188                assert_eq!(runner.state.candidates[1].commits, 1);
14189            }
14190        }
14191    }
14192
14193    #[tokio::test]
14194    #[cfg(unix)]
14195    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14196        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14197            let (_tmp, mut runner) = preservation_resume_fixture().await;
14198            runner.state.candidates.retain(|c| c.label == 'A');
14199            runner.state.candidates[0].empty = false;
14200            runner.state.candidates[0].summary.clear();
14201            runner.state.tally = None;
14202            runner.state.judge_skipped = false;
14203            runner.state.status = RunStatus::Implementing;
14204            // Force the real post-spawn watchdog, independently of available
14205            // disk space. Dirty/committed work predates its first measurement.
14206            runner.state.config.disk.min_free_bytes = u64::MAX;
14207            let wt = runner.state.candidates[0].worktree.clone();
14208            std::fs::write(
14209                wt.join("interrupted-source"),
14210                "retain implementation work\n",
14211            )
14212            .unwrap();
14213            if committed {
14214                git::rescue_commit(&wt, "implementation checkpoint")
14215                    .await
14216                    .unwrap();
14217            }
14218            let lock = git::git(
14219                &wt,
14220                &[
14221                    "rev-parse",
14222                    "--path-format=absolute",
14223                    "--git-path",
14224                    "index.lock",
14225                ],
14226            )
14227            .await
14228            .unwrap();
14229            if block_rescue {
14230                std::fs::write(&lock, "fixture-owned lock").unwrap();
14231            }
14232            let mut implementer = spec("alpha");
14233            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14234            runner.roles.implementers = vec![implementer];
14235            runner.state.save().unwrap();
14236            let result = runner.implement().await;
14237            assert_eq!(result.is_err(), block_rescue);
14238            assert_eq!(
14239                runner.state.resource_block.as_ref().unwrap().kind,
14240                "disk pressure"
14241            );
14242            assert!(
14243                runner
14244                    .state
14245                    .events
14246                    .iter()
14247                    .any(|e| e.message.contains("agent execution interrupted"))
14248            );
14249            assert!(
14250                runner
14251                    .state
14252                    .dir()
14253                    .join("artifacts/candidate-A-recovery/staged.patch")
14254                    .exists()
14255            );
14256            if block_rescue {
14257                let saved = RunState::load(&runner.state.id).unwrap();
14258                assert!(saved.candidate_preservation.contains_key(&'A'));
14259                assert!(!saved.candidates[0].empty);
14260                assert_eq!(
14261                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14262                    "retain implementation work\n"
14263                );
14264                std::fs::remove_file(lock).unwrap();
14265                runner.state = saved;
14266            } else {
14267                assert_eq!(runner.state.status, RunStatus::Blocked);
14268                assert_eq!(runner.state.candidates[0].commits, 1);
14269            }
14270            // Recovery must save the candidate before deciding whether another
14271            // implementation is needed. No implementer is available on retry.
14272            runner.roles.implementers.clear();
14273            runner.state.config.disk.min_free_bytes = 0;
14274            runner.state.resource_block = None;
14275            runner.implement().await.unwrap();
14276            assert!(runner.state.candidate_preservation.is_empty());
14277            assert_eq!(runner.state.candidates[0].commits, 1);
14278            assert!(!runner.state.candidates[0].empty);
14279            assert!(git::is_clean(&wt).await.unwrap());
14280            assert!(
14281                crate::run::read_artifact(&runner.state, "cand-A.patch")
14282                    .unwrap()
14283                    .contains("retain implementation work")
14284            );
14285        }
14286    }
14287
14288    #[tokio::test]
14289    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14290        let (_tmp, mut runner) = preservation_resume_fixture().await;
14291        // Schema 15 already confirmed the unfolded no-change result.
14292        runner.implement().await.unwrap();
14293        assert!(runner.state.candidate_preservation.is_empty());
14294        assert!(
14295            !runner
14296                .state
14297                .dir()
14298                .join("artifacts/candidate-A-recovery")
14299                .exists()
14300        );
14301        runner.fold_losers().await.unwrap();
14302        assert!(!runner.state.candidates[0].worktree.exists());
14303        for schema in [14, crate::run::SCHEMA] {
14304            runner.state.schema = schema;
14305            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14306            runner.state.save().unwrap();
14307            runner.state = RunState::load(&runner.state.id).unwrap();
14308            runner.implement().await.unwrap();
14309            assert!(runner.state.candidate_preservation.is_empty());
14310            assert!(runner.state.candidates[0].folded);
14311            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14312        }
14313    }
14314
14315    #[tokio::test]
14316    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14317        let (_tmp, mut runner) = preservation_resume_fixture().await;
14318        let wt = runner.state.candidates[0].worktree.clone();
14319        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14320        runner.state.schema = 14;
14321        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14322        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14323        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14324        for _ in 0..2 {
14325            assert!(runner.implement().await.is_err());
14326            assert_eq!(runner.state.status, RunStatus::Blocked);
14327            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14328            assert!(
14329                runner.state.candidates[0].empty,
14330                "old verdict must remain unusable"
14331            );
14332            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14333            assert!(runner.state.judge_skipped);
14334            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14335            assert!(
14336                crate::run::read_artifact(&runner.state, "cand-A.patch")
14337                    .unwrap()
14338                    .contains("owned edits")
14339            );
14340            assert!(wt.exists());
14341            assert_eq!(
14342                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14343                    .await
14344                    .unwrap(),
14345                1
14346            );
14347            runner.state = RunState::load(&runner.state.id).unwrap();
14348        }
14349    }
14350
14351    #[tokio::test]
14352    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14353        let (_tmp, mut runner) = preservation_resume_fixture().await;
14354        let wt = runner.state.candidates[0].worktree.clone();
14355        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14356        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14357        let index_lock = PathBuf::from(
14358            git::git(
14359                &wt,
14360                &[
14361                    "rev-parse",
14362                    "--path-format=absolute",
14363                    "--git-path",
14364                    "index.lock",
14365                ],
14366            )
14367            .await
14368            .unwrap(),
14369        );
14370        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14371        runner.state.schema = 14;
14372        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14373        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14374        assert!(runner.implement().await.is_err());
14375        assert!(runner.after_implement().is_err());
14376        assert!(runner.fold_losers().await.is_err());
14377        assert_eq!(
14378            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14379            "failed seat edits"
14380        );
14381        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14382        runner.state = RunState::load(&runner.state.id).unwrap();
14383        std::fs::remove_file(index_lock).unwrap();
14384        // Recovery finds changes that the old uncontested result never judged.
14385        assert!(runner.implement().await.is_err());
14386        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14387        assert!(runner.state.candidates[0].empty);
14388        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14389        assert!(
14390            crate::run::read_artifact(&runner.state, "cand-A.patch")
14391                .unwrap()
14392                .contains("failed seat edits")
14393        );
14394        assert!(wt.exists());
14395    }
14396
14397    #[tokio::test]
14398    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14399        let (_tmp, mut runner) = preservation_resume_fixture().await;
14400        let repo = runner.state.repo.clone();
14401        let common = git::git(
14402            &repo,
14403            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14404        )
14405        .await
14406        .unwrap();
14407        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14408        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14409        assert!(runner.fold_losers().await.is_err());
14410        assert!(!runner.state.candidates[0].worktree.exists());
14411        assert!(!runner.state.candidates[0].folded);
14412        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14413        runner.state = RunState::load(&runner.state.id).unwrap();
14414        std::fs::remove_file(branch_lock).unwrap();
14415        let judge = runner.state.worktree_root().join("judge-1");
14416        git::worktree_add_detached(&repo, &judge, "main")
14417            .await
14418            .unwrap();
14419        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14420        runner.roles.judges = vec![spec("judge")];
14421        assert!(runner.fold_losers().await.is_err());
14422        runner.state = RunState::load(&runner.state.id).unwrap();
14423        assert!(
14424            runner.state.candidates[0].folded,
14425            "candidate progress saved before judge failure"
14426        );
14427        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14428        assert!(judge.join("uncommitted").exists());
14429        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14430        runner.fold_losers().await.unwrap();
14431        assert!(!judge.exists());
14432        runner.fold_losers().await.unwrap();
14433    }
14434
14435    #[tokio::test]
14436    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14437        let (_tmp, mut runner) = preservation_resume_fixture().await;
14438        runner.state.candidates.retain(|c| c.label == 'B');
14439        runner.state.tally = None;
14440        runner.state.judge_skipped = false;
14441        runner.state.prep_complete = Some(true);
14442        runner.state.config.graph.advise = false;
14443        runner.state.config.graph.selected_implementer = Some("alpha".into());
14444        let wt = runner.state.candidates[0].worktree.clone();
14445        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14446        runner
14447            .state
14448            .candidate_preservation
14449            .insert('B', "awaiting persistence".into());
14450        let index_lock = git::git(
14451            &wt,
14452            &[
14453                "rev-parse",
14454                "--path-format=absolute",
14455                "--git-path",
14456                "index.lock",
14457            ],
14458        )
14459        .await
14460        .unwrap();
14461        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14462        runner.state.save().unwrap();
14463        assert!(runner.execute_graph().await.is_err());
14464        assert_eq!(runner.state.status, RunStatus::Blocked);
14465        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14466        std::fs::remove_file(index_lock).unwrap();
14467        // Park at the next graph boundary: this tests the real selected-seat
14468        // hold guard without dispatching reviewers or publishing anything.
14469        for _ in 0..2 {
14470            runner.state = RunState::load(&runner.state.id).unwrap();
14471            runner.pause.park();
14472            runner.execute_graph().await.unwrap();
14473            assert_eq!(runner.state.status, RunStatus::Implementing);
14474            assert!(runner.state.candidate_preservation.is_empty());
14475            assert_eq!(runner.state.candidates[0].commits, 2);
14476            assert!(runner.state.tally.is_none());
14477        }
14478        runner.implement().await.unwrap();
14479        assert_eq!(runner.state.status, RunStatus::Judging);
14480        // A genuine selected-implementer hold must still require release.
14481        runner.state.status = RunStatus::Blocked;
14482        runner.state.candidates[0].failed = Some("implementation failed".into());
14483        runner.state.save().unwrap();
14484        let error = runner.execute_graph().await.unwrap_err();
14485        assert!(
14486            error
14487                .to_string()
14488                .contains("selected implementation is held")
14489        );
14490    }
14491
14492    #[tokio::test]
14493    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14494        let (_tmp, mut runner) = preservation_resume_fixture().await;
14495        let repo = runner.state.repo.clone();
14496        let candidate = runner.state.candidates[0].clone();
14497        // Remove only the disposable fixture directory, leaving Git's linked
14498        // worktree registration and branch intact, as in a vanished checkout.
14499        let registered = git::worktree_holding(&repo, &candidate.branch)
14500            .await
14501            .unwrap()
14502            .unwrap();
14503        assert_eq!(
14504            registered.canonicalize().unwrap(),
14505            candidate.worktree.canonicalize().unwrap()
14506        );
14507        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14508        assert_eq!(
14509            git::worktree_holding(&repo, &candidate.branch)
14510                .await
14511                .unwrap(),
14512            Some(registered)
14513        );
14514        assert!(
14515            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14516            "the stale registration initially keeps the branch checked out"
14517        );
14518        runner.fold_losers().await.unwrap();
14519        assert!(runner.state.candidates[0].folded);
14520        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14521        assert_eq!(
14522            git::worktree_holding(&repo, &candidate.branch)
14523                .await
14524                .unwrap(),
14525            None
14526        );
14527        runner.state = RunState::load(&runner.state.id).unwrap();
14528        runner.fold_losers().await.unwrap();
14529    }
14530
14531    #[tokio::test]
14532    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14533        let (_tmp, mut runner) = preservation_resume_fixture().await;
14534        let repo = runner.state.repo.clone();
14535        let candidate = runner.state.candidates[0].clone();
14536        let winner = runner.state.candidates[1].clone();
14537        let registered = git::worktree_holding(&repo, &candidate.branch)
14538            .await
14539            .unwrap()
14540            .unwrap();
14541        assert_eq!(
14542            registered.canonicalize().unwrap(),
14543            candidate.worktree.canonicalize().unwrap()
14544        );
14545        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14546        assert_eq!(
14547            git::worktree_holding(&repo, &candidate.branch)
14548                .await
14549                .unwrap(),
14550            Some(registered)
14551        );
14552        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14553
14554        for _ in 0..2 {
14555            fold_run(&mut runner.state, false, &crate::run::home())
14556                .await
14557                .unwrap();
14558            runner.state = RunState::load(&runner.state.id).unwrap();
14559            assert!(runner.state.candidates[0].folded);
14560            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14561            assert_eq!(
14562                git::worktree_holding(&repo, &candidate.branch)
14563                    .await
14564                    .unwrap(),
14565                None
14566            );
14567            assert!(!runner.state.candidates[1].folded);
14568            assert!(winner.worktree.exists());
14569            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14570        }
14571    }
14572
14573    #[tokio::test]
14574    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14575        ask_test_home();
14576        let tmp = tempfile::tempdir().unwrap();
14577        let repo = tmp.path().join("repo");
14578        std::fs::create_dir(&repo).unwrap();
14579        init_repo(&repo);
14580        let mut runner = runner_at(RunStatus::Implementing);
14581        runner.state.repo = repo.clone();
14582        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14583        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14584        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14585            let branch = format!("candidate-{label}");
14586            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14587            git::worktree_add_branch(&repo, &wt, &branch, "main")
14588                .await
14589                .unwrap();
14590            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14591            git::git(&wt, &["add", "README.md"]).await.unwrap();
14592            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14593            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14594            runner.state.candidates.push(Candidate {
14595                index,
14596                label,
14597                agent: "alpha".to_owned(),
14598                branch,
14599                worktree: wt,
14600                summary: "completed implementation".to_owned(),
14601                stat: String::new(),
14602                files: 0,
14603                commits: 0,
14604                empty: false,
14605                failed: None,
14606                verified_noop: None,
14607                duration_ms: 42,
14608                folded: false,
14609            });
14610            runner
14611                .state
14612                .candidate_preservation
14613                .insert(label, "awaiting persistence".to_owned());
14614        }
14615        let wt = runner.state.candidates[0].worktree.clone();
14616        let index_lock = git::git(
14617            &wt,
14618            &[
14619                "rev-parse",
14620                "--path-format=absolute",
14621                "--git-path",
14622                "index.lock",
14623            ],
14624        )
14625        .await
14626        .unwrap();
14627        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14628        runner.state.save().unwrap();
14629        assert!(runner.persist_candidates().await.is_err());
14630        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14631        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14632        assert_eq!(
14633            runner.state.candidates[1].commits, 1,
14634            "other results are still collected"
14635        );
14636        assert!(
14637            !runner.state.candidates[0].empty,
14638            "failed rescue is not no change"
14639        );
14640        assert_eq!(
14641            runner.state.candidates[0].summary,
14642            "completed implementation"
14643        );
14644        assert!(runner.after_implement().is_err());
14645        let home = tmp.path().join("fold-home");
14646        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14647        assert!(wt.join("new-file").exists());
14648        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14649        assert!(recovery.join("staged.patch").exists());
14650        assert!(recovery.join("unstaged.patch").exists());
14651        assert_eq!(
14652            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14653            "untracked edits\n"
14654        );
14655        runner.state.begin_pass();
14656        runner.state.end_pass_parked(ParkCause::Upgrade {
14657            from: "1.0".into(),
14658            to: Some("1.1".into()),
14659        });
14660        runner.state.save().unwrap();
14661        runner.state = RunState::load(&runner.state.id).unwrap();
14662        assert!(matches!(
14663            runner.state.passes.last().unwrap().end,
14664            Some(crate::run::PassEnd::Parked {
14665                cause: ParkCause::Upgrade { .. }
14666            })
14667        ));
14668        std::fs::remove_file(index_lock).unwrap();
14669        // No implementer is configured: a persistence retry cannot dispatch a seat.
14670        runner.implement().await.unwrap();
14671        assert!(runner.state.candidate_preservation.is_empty());
14672        assert!(
14673            runner
14674                .state
14675                .candidates
14676                .iter()
14677                .all(|c| c.commits == 1 && !c.empty)
14678        );
14679        assert!(git::is_clean(&wt).await.unwrap());
14680        assert!(
14681            crate::run::read_artifact(&runner.state, "cand-A.patch")
14682                .unwrap()
14683                .contains("later tracked edits")
14684        );
14685    }
14686
14687    #[tokio::test]
14688    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14689        ask_test_home();
14690        let tmp = tempfile::tempdir().unwrap();
14691        let repo = tmp.path().join("repo");
14692        std::fs::create_dir(&repo).unwrap();
14693        init_repo(&repo);
14694        let wt = tmp.path().join("candidate");
14695        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14696            .await
14697            .unwrap();
14698        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14699        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14700        assert!(wt.join("uncommitted").exists());
14701        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14702        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14703        let mut runner = runner_at(RunStatus::Implementing);
14704        runner.state.repo = repo.clone();
14705        runner.state.base_commit = "missing-base".to_owned();
14706        runner.state.candidates.push(Candidate {
14707            index: 0,
14708            label: 'A',
14709            agent: "alpha".into(),
14710            branch: "candidate".into(),
14711            worktree: wt.clone(),
14712            summary: "completed".into(),
14713            stat: String::new(),
14714            files: 0,
14715            commits: 0,
14716            empty: false,
14717            failed: None,
14718            verified_noop: None,
14719            duration_ms: 1,
14720            folded: false,
14721        });
14722        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14723        assert!(
14724            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14725                .await
14726                .is_err()
14727        );
14728        assert!(!runner.state.candidates[0].folded);
14729        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14730        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14731        runner
14732            .state
14733            .candidate_preservation
14734            .insert('A', "pending".into());
14735        assert!(runner.persist_candidates().await.is_err());
14736        assert!(!runner.state.candidates[0].empty);
14737        assert!(!runner.state.candidates[0].folded);
14738        assert!(wt.exists());
14739        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14740        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14741        std::fs::create_dir_all(&patch_path).unwrap();
14742        assert!(
14743            runner.persist_candidates().await.is_err(),
14744            "artifact failure stays pending"
14745        );
14746        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14747        assert!(!runner.state.candidates[0].empty);
14748        std::fs::remove_dir(patch_path).unwrap();
14749        runner.persist_candidates().await.unwrap();
14750        assert!(
14751            runner.state.candidates[0].empty,
14752            "only successful inspection confirms no change"
14753        );
14754    }
14755
14756    fn init_repo(dir: &Path) {
14757        let run = |args: &[&str]| {
14758            let out = std::process::Command::new("git")
14759                .args(args)
14760                .current_dir(dir)
14761                .quiet()
14762                .output()
14763                .expect("spawn git");
14764            assert!(
14765                out.status.success(),
14766                "git {args:?} failed: {}",
14767                String::from_utf8_lossy(&out.stderr)
14768            );
14769        };
14770        run(&["init", "-b", "main"]);
14771        run(&["config", "user.name", "magi test"]);
14772        run(&["config", "user.email", "magi@example.com"]);
14773        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14774        run(&["add", "-A"]);
14775        run(&["commit", "-m", "init"]);
14776    }
14777
14778    // `settle_questions` is what closes the ghost the phone showed: a run's
14779    // seat asked something, the run then ended, and nothing was left to
14780    // abandon the question it left `open`. `HOME` is a process-wide
14781    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
14782    // first time it runs in the binary — every test below still reaches the
14783    // same directory whichever call won, and each gets its own run id from
14784    // `RunState::new`, so they never collide there.
14785    fn ask_test_home() {
14786        crate::run::pin_test_home();
14787    }
14788
14789    /// A minimal, git-free `Runner` at a given status — `settle_questions`
14790    /// reads nothing else off it.
14791    fn runner_at(status: RunStatus) -> Runner {
14792        let mut state = RunState::new(
14793            PathBuf::from("/nonexistent/repo"),
14794            "main".to_owned(),
14795            "deadbeef".to_owned(),
14796            "task".to_owned(),
14797            Config::default(),
14798        );
14799        // These fixtures share one home and save concurrently. Short clock
14800        // IDs can collide, making an ownership test read another test's run.
14801        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14802        state.id = format!(
14803            "20000101-000000-{:04x}",
14804            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14805        );
14806        state.status = status;
14807        Runner {
14808            state,
14809            roles: ResolvedRoles {
14810                implementers: Vec::new(),
14811                judges: Vec::new(),
14812                reviewers: Vec::new(),
14813                fixer: None,
14814                conductor: conductor(),
14815                implementer_roster: Vec::new(),
14816                judge_roster: Vec::new(),
14817                reviewer_roster: Vec::new(),
14818            },
14819            sem: Arc::new(Semaphore::new(1)),
14820            pause: Pause::new(),
14821            interrupt: Pause::new(),
14822        }
14823    }
14824
14825    #[tokio::test]
14826    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14827        crate::run::pin_test_home();
14828        let mut runner = runner_at(RunStatus::Gating);
14829        runner.state.save().unwrap();
14830        let id = runner.state.id.clone();
14831        let passes = runner.state.passes.len();
14832        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14833        let mut claiming = Box::pin(runner.claim_driver());
14834        std::future::poll_fn(|cx| {
14835            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14836            std::task::Poll::Ready(())
14837        })
14838        .await;
14839        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14840        drop(owner);
14841        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14842            .await
14843            .unwrap()
14844            .unwrap();
14845        assert_eq!(execution.runner.state.passes.len(), passes + 1);
14846        assert!(execution.runner.state.resource_block.is_none());
14847        assert_eq!(
14848            execution.runner.state.liveness(false),
14849            crate::run::Liveness::Live
14850        );
14851        drop(execution);
14852        assert!(RunState::load(&id).unwrap().driver_exited);
14853    }
14854
14855    #[tokio::test]
14856    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14857        crate::run::pin_test_home();
14858        let mut runner = runner_at(RunStatus::Gating);
14859        runner.state.save().unwrap();
14860        // run.json can commit, but the subsequent lease metadata write fails.
14861        // Cleanup must run after releasing the short ownership lock.
14862        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14863        assert!(runner.claim_driver().await.is_err());
14864        assert!(runner.state.driver_exited);
14865        assert_eq!(runner.state.driver_pid, None);
14866        let disk = RunState::load(&runner.state.id).unwrap();
14867        assert!(disk.driver_exited);
14868        assert_eq!(disk.driver_pid, None);
14869        assert_eq!(disk.status, RunStatus::Gating);
14870    }
14871
14872    #[tokio::test]
14873    async fn an_early_graph_error_releases_its_driver() {
14874        crate::run::pin_test_home();
14875        let mut runner = runner_at(RunStatus::Blocked);
14876        runner.state.config.graph.selected_implementer = Some("selected".into());
14877        runner.state.save().unwrap();
14878        assert!(runner.execute().await.is_err());
14879        let disk = RunState::load(&runner.state.id).unwrap();
14880        assert!(disk.driver_exited);
14881        assert_eq!(disk.driver_pid, None);
14882        assert_eq!(disk.status, RunStatus::Blocked);
14883    }
14884
14885    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14886        crate::run::MergeRepair {
14887            question: "repair-question".into(),
14888            head: "old-head".into(),
14889            pr: "https://github.com/o/r/pull/42".into(),
14890            reviews: vec![review_round(false, 1, 1, 1, false, true)],
14891            gate: vec![outcome(Some(0), false)],
14892            gate_head: Some("old-head".into()),
14893            gate_fixes: Vec::new(),
14894            pre_gate: Vec::new(),
14895            phase,
14896            failure: None,
14897        }
14898    }
14899
14900    #[tokio::test]
14901    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14902        crate::run::pin_test_home();
14903        let mut runner = runner_at(RunStatus::Landing);
14904        runner.state.merge_repairs.push(merge_repair_request(
14905            crate::run::MergeRepairPhase::Requested,
14906        ));
14907        runner.state.save().unwrap();
14908        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14909        assert!(
14910            runner
14911                .execute()
14912                .await
14913                .unwrap_err()
14914                .to_string()
14915                .contains("already running")
14916        );
14917        assert_eq!(
14918            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14919            crate::run::MergeRepairPhase::Requested
14920        );
14921        drop(claim);
14922        let mut disk = runner.state.clone();
14923        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14924        disk.save().unwrap();
14925        assert!(
14926            runner
14927                .execute()
14928                .await
14929                .unwrap_err()
14930                .to_string()
14931                .contains("decision changed")
14932        );
14933        assert_eq!(
14934            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14935            crate::run::MergeRepairPhase::Complete
14936        );
14937    }
14938
14939    #[test]
14940    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14941        crate::run::pin_test_home();
14942        let mut runner = runner_at(RunStatus::Landing);
14943        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14944        runner.state.reviews = request.reviews.clone();
14945        runner.state.gate = request.gate.clone();
14946        runner.state.gate_ran = true;
14947        runner.state.land_approval = Some(crate::run::LandApproval {
14948            question: "old".into(),
14949            head: "old-head".into(),
14950        });
14951        runner.state.merge_repairs.push(request);
14952        runner.begin_merge_repair_review(0).unwrap();
14953        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14954        assert!(!runner.state.gate_status().ok());
14955        assert!(runner.state.land_approval.is_none());
14956        let disk = RunState::load(&runner.state.id).unwrap();
14957        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14958        assert!(disk.merge_repairs[0].gate[0].ok());
14959        assert_eq!(
14960            disk.merge_repairs[0].phase,
14961            crate::run::MergeRepairPhase::Reviewing
14962        );
14963        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14964        assert!(disk.reviews.is_empty());
14965    }
14966
14967    #[tokio::test]
14968    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14969        crate::run::pin_test_home();
14970        for phase in [
14971            crate::run::MergeRepairPhase::Fixing,
14972            crate::run::MergeRepairPhase::Requested,
14973        ] {
14974            let mut runner = runner_at(RunStatus::Landing);
14975            runner.state.config.graph.land_rounds = 0;
14976            runner.state.merge_repairs.push(merge_repair_request(phase));
14977            runner.state.save().unwrap();
14978            assert!(!runner.repair_merge_review().await.unwrap());
14979            assert_eq!(runner.state.status, RunStatus::Blocked);
14980            assert_eq!(
14981                runner.state.merge_repairs[0].phase,
14982                crate::run::MergeRepairPhase::Failed
14983            );
14984            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14985            assert!(
14986                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14987                    "interrupted"
14988                } else {
14989                    "budget exhausted"
14990                })
14991            );
14992            assert!(runner.state.merge.is_none());
14993        }
14994    }
14995
14996    /// `park_here` folding in the reason `Pause::park_because` recorded -
14997    /// this is what lets an operator reading a run's events tell an
14998    /// interrupt-driven park from an ordinary shutdown park.
14999    #[test]
15000    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15001        crate::run::pin_test_home();
15002        let mut runner = runner_at(RunStatus::Implementing);
15003        let interrupt = Pause::new();
15004        runner.watch_interrupt(interrupt.clone());
15005
15006        interrupt.park_because("task a1b2 asked to run first");
15007
15008        assert!(runner.park_here().expect("park_here"));
15009        assert!(runner.state.parked);
15010        let last = runner.state.events.last().expect("a park event");
15011        assert_eq!(last.node, "park");
15012        assert!(
15013            last.message.contains("task a1b2 asked to run first"),
15014            "expected the interrupt reason in {:?}",
15015            last.message
15016        );
15017    }
15018
15019    /// The cause is stored on the pass when it parks, and the first cause a
15020    /// shared pause was given is the one that stays.
15021    #[test]
15022    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15023        crate::run::pin_test_home();
15024        let mut runner = runner_at(RunStatus::Implementing);
15025        runner.state.begin_pass();
15026        let shutdown = Pause::new();
15027        runner.on_pause(shutdown.clone());
15028        let upgrade = ParkCause::Upgrade {
15029            from: "0.121.0".to_owned(),
15030            to: Some("0.121.1".to_owned()),
15031        };
15032        shutdown.park_for(upgrade.clone());
15033        shutdown.park_for(ParkCause::Operator);
15034
15035        assert!(runner.park_here().expect("park_here"));
15036        assert_eq!(
15037            runner.state.passes[0].end,
15038            Some(crate::run::PassEnd::Parked { cause: upgrade })
15039        );
15040        let last = runner.state.events.last().expect("a park event");
15041        assert!(last.message.contains("0.121.0"), "{}", last.message);
15042    }
15043
15044    #[tokio::test]
15045    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15046        crate::run::pin_test_home();
15047        let mut runner = runner_at(RunStatus::Reviewing);
15048        runner.state.save().unwrap();
15049        let mut execution = runner.claim_driver().await.expect("first execution");
15050        assert_eq!(execution.runner.state.passes.len(), 1);
15051        execution
15052            .driver
15053            .finish(&mut execution.runner.state)
15054            .await
15055            .unwrap();
15056    }
15057
15058    #[tokio::test]
15059    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15060        crate::run::pin_test_home();
15061        let mut runner = runner_at(RunStatus::Reviewing);
15062        runner.state.config.agents = vec![conductor()];
15063        runner.state.config.graph.timeout_resource_wait = 0;
15064        runner.state.save().unwrap();
15065        let id = runner.state.id.clone();
15066        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15067        assert!(runner.claim_driver().await.is_err());
15068        drop(owner);
15069        drop(runner);
15070        let mut resumed = Runner::resume(&id).unwrap();
15071        assert_eq!(resumed.state.driver_claimed, Some(false));
15072        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15073        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15074        let mut execution = resumed
15075            .claim_driver()
15076            .await
15077            .expect("retry initial claim after reload");
15078        assert_eq!(execution.runner.state.driver_claimed, Some(true));
15079        assert_eq!(execution.runner.state.passes.len(), 1);
15080        execution
15081            .driver
15082            .finish(&mut execution.runner.state)
15083            .await
15084            .unwrap();
15085    }
15086
15087    #[tokio::test]
15088    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15089        crate::run::pin_test_home();
15090        let mut runner = runner_at(RunStatus::Reviewing);
15091        runner.state.config.agents = vec![conductor()];
15092        runner.state.save().unwrap();
15093        let id = runner.state.id.clone();
15094        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15095        let mut resumed = Runner::resume(&id).unwrap();
15096        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15097        assert!(resumed.claim_driver().await.is_err());
15098        assert!(RunState::load(&id).unwrap().passes.is_empty());
15099        drop(lease);
15100        let mut execution = resumed.claim_driver().await.unwrap();
15101        execution
15102            .driver
15103            .finish(&mut execution.runner.state)
15104            .await
15105            .unwrap();
15106    }
15107
15108    #[tokio::test]
15109    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15110        crate::run::pin_test_home();
15111        let mut runner = runner_at(RunStatus::Reviewing);
15112        let state = &mut runner.state;
15113        state.config.agents = vec![conductor()];
15114        state.driver_claimed = None;
15115        state.driver_pid = Some(std::process::id());
15116        state.driver_started_at = None;
15117        state.begin_pass();
15118        state.save().unwrap();
15119        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15120        std::fs::write(
15121            state.dir().join("driver.json"),
15122            serde_json::to_vec(&serde_json::json!({
15123                "pid": state.driver_pid, "started_at": null,
15124                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15125            }))
15126            .unwrap(),
15127        )
15128        .unwrap();
15129        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15130        drop(lease);
15131        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15132        let mut resumed = Runner::resume(&state.id).unwrap();
15133        assert!(!resumed.state.driver_exited);
15134        let mut execution = resumed
15135            .claim_driver()
15136            .await
15137            .expect("lease proves previous execution ended");
15138        assert_eq!(execution.runner.state.passes.len(), 2);
15139        execution
15140            .driver
15141            .finish(&mut execution.runner.state)
15142            .await
15143            .unwrap();
15144    }
15145
15146    #[tokio::test]
15147    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15148        crate::run::pin_test_home();
15149        let mut runner = runner_at(RunStatus::Reviewing);
15150        let mut json = serde_json::to_value(&runner.state).unwrap();
15151        json.as_object_mut().unwrap().remove("driver_claimed");
15152        runner.state = serde_json::from_value(json).unwrap();
15153        assert_eq!(runner.state.driver_claimed, None);
15154        runner.state.save().unwrap();
15155        let mut execution = runner
15156            .claim_driver()
15157            .await
15158            .expect("the execution lease, not a missing marker, excludes a second driver");
15159        assert_eq!(execution.runner.state.passes.len(), 1);
15160        execution
15161            .driver
15162            .finish(&mut execution.runner.state)
15163            .await
15164            .unwrap();
15165    }
15166
15167    #[tokio::test]
15168    async fn driver_admission_refuses_a_proven_live_driver() {
15169        crate::run::pin_test_home();
15170        let mut runner = runner_at(RunStatus::Reviewing);
15171        let pid = std::process::id();
15172        runner.state.driver_pid = Some(pid);
15173        runner.state.driver_started_at = crate::proc::process_started_at(pid);
15174        assert!(runner.state.driver_started_at.is_some());
15175        runner.state.save().unwrap();
15176        let error = runner
15177            .claim_driver()
15178            .await
15179            .err()
15180            .expect("a live driver must refuse");
15181        assert!(error.to_string().contains("live driver"), "{error:#}");
15182        let saved = RunState::load(&runner.state.id).unwrap();
15183        assert_eq!(saved.driver_pid, Some(pid));
15184        assert_eq!(saved.status, RunStatus::Reviewing);
15185        assert!(saved.passes.is_empty());
15186    }
15187
15188    #[test]
15189    fn parking_resource_halt_persists_the_pass_once() {
15190        crate::run::pin_test_home();
15191        let mut runner = runner_at(RunStatus::Reviewing);
15192        runner.state.begin_pass();
15193        runner.state.resource_block = Some(crate::run::ResourceBlock {
15194            node: "fix".into(),
15195            kind: "shared build cache".into(),
15196            detail: "cancelled".into(),
15197            at: Timestamp::now(),
15198            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15199        });
15200        let cause = ParkCause::Upgrade {
15201            from: "old".into(),
15202            to: Some("new".into()),
15203        };
15204        runner.pause.park_for(cause.clone());
15205        // Both direct fixer exits and graph resource boundaries must complete
15206        // parking, rather than returning a stale Reviewing record.
15207        runner.stop_for_resource().unwrap();
15208        assert!(runner.resource_halt().unwrap());
15209        assert!(runner.park_here().unwrap());
15210        let saved = RunState::load(&runner.state.id).unwrap();
15211        assert!(saved.parked);
15212        assert_eq!(saved.status, RunStatus::Reviewing);
15213        assert!(saved.passes[0].ended_at.is_some());
15214        assert_eq!(
15215            saved.passes[0].end,
15216            Some(crate::run::PassEnd::Parked {
15217                cause: cause.clone()
15218            })
15219        );
15220        assert_eq!(
15221            saved.resource_block.unwrap().cause,
15222            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15223        );
15224        assert_eq!(
15225            saved
15226                .events
15227                .iter()
15228                .filter(|event| event.node == "park")
15229                .count(),
15230            1
15231        );
15232    }
15233
15234    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
15235    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
15236    /// cleared) must not make a *different* run - one only watching its own,
15237    /// unshared interrupt `Pause` - see itself as parked. If a future change
15238    /// ever collapsed these back into one handle, the interrupt scheduler
15239    /// would park every run for the rest of the daemon's life, not just the
15240    /// one it meant to interrupt.
15241    #[test]
15242    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15243        crate::run::pin_test_home();
15244        let mut runner = runner_at(RunStatus::Implementing);
15245        let shutdown = Pause::new();
15246        runner.on_pause(shutdown.clone());
15247        let interrupt = Pause::new();
15248        runner.watch_interrupt(interrupt.clone());
15249
15250        // Nobody has asked for anything yet.
15251        assert!(!runner.park_here().expect("park_here"));
15252        assert!(!runner.state.parked);
15253
15254        // Only the interrupt handle fires; the shutdown handle stays clear.
15255        interrupt.park_because("test");
15256        assert!(!shutdown.parked());
15257        assert!(runner.park_here().expect("park_here"));
15258    }
15259
15260    /// The property every prior attempt at this feature failed to pin down:
15261    /// asking a run to park while one of its nodes has a real, in-flight
15262    /// async operation running (an agent call, in production) must not cut
15263    /// that operation short. `park_here` is only ever consulted *between*
15264    /// `execute`'s node calls - see its own doc - so nothing inside a node
15265    /// can observe a park request until the node itself returns. This proves
15266    /// that structurally, with real `tokio` concurrency and a channel
15267    /// handshake (never a sleep, which would only prove "usually", not
15268    /// "cannot"): the "node" below reports that it has genuinely started,
15269    /// and only then is the park requested; the node still has to be told to
15270    /// finish before `park_here` is ever called, exactly mirroring every
15271    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
15272    /// in `execute`.
15273    #[tokio::test]
15274    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15275        crate::run::pin_test_home();
15276        let mut runner = runner_at(RunStatus::Implementing);
15277        let interrupt = Pause::new();
15278        runner.watch_interrupt(interrupt.clone());
15279
15280        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15281        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15282
15283        // Stands in for one node's in-flight agent call: it proves it has
15284        // genuinely started, then blocks - exactly as a spawned CLI process
15285        // does - until told to finish.
15286        let node = async move {
15287            started_tx.send(()).expect("send started");
15288            finish_rx.await.expect("recv finish");
15289            "node finished"
15290        };
15291
15292        let interrupter = async move {
15293            started_rx.await.expect("recv started");
15294            // The call is now genuinely in flight. Ask it to park.
15295            interrupt.park_because("higher-priority task waiting");
15296            // Nothing the node does can observe this yet - there is no
15297            // check inside it, by construction - so let the executor run
15298            // anything pending and then let the node finish on its own.
15299            tokio::task::yield_now().await;
15300            finish_tx.send(()).expect("send finish");
15301        };
15302
15303        let (node_result, ()) = tokio::join!(node, interrupter);
15304        assert_eq!(
15305            node_result, "node finished",
15306            "the in-flight call ran to completion"
15307        );
15308
15309        // Only now, at the boundary the real `execute` would check right
15310        // after this node, does the park take effect.
15311        assert!(runner.park_here().expect("park_here"));
15312        assert!(runner.state.parked);
15313    }
15314
15315    /// A run parked mid-competition carries every field it had accumulated
15316    /// through the exact same disk round-trip an ordinary resume uses -
15317    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
15318    /// Nothing about parking for an interrupt is a special case of that path;
15319    /// this is what proves it rather than assuming it.
15320    #[test]
15321    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15322        crate::run::pin_test_home();
15323        let mut runner = runner_at(RunStatus::Judging);
15324        // `Runner::resume` re-resolves roles from the saved config, which
15325        // refuses an empty roster - give it the same minimal one `conductor`
15326        // itself uses.
15327        runner.state.config.agents = vec![conductor()];
15328        runner.state.candidates = vec![Candidate {
15329            index: 0,
15330            label: 'A',
15331            agent: "alpha".to_owned(),
15332            branch: "magi/x/A".to_owned(),
15333            worktree: PathBuf::from("/nonexistent/worktree"),
15334            summary: "did the thing".to_owned(),
15335            stat: "1 file changed".to_owned(),
15336            files: 1,
15337            commits: 1,
15338            empty: false,
15339            failed: None,
15340            verified_noop: None,
15341            duration_ms: 1234,
15342            folded: false,
15343        }];
15344        let run_id = runner.state.id.clone();
15345
15346        let interrupt = Pause::new();
15347        runner.watch_interrupt(interrupt.clone());
15348        interrupt.park_because("task c3d4 asked to run first");
15349        assert!(runner.park_here().expect("park_here"));
15350
15351        let resumed = Runner::resume(&run_id).expect("resume");
15352        assert_eq!(resumed.state.candidates.len(), 1);
15353        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15354        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15355        assert_eq!(resumed.state.status, runner.state.status);
15356        assert!(
15357            resumed.state.parked,
15358            "still parked until `execute` actually walks the graph again"
15359        );
15360        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15361    }
15362
15363    /// A fresh open question on `run`, stored and handed back for assertions.
15364    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15365        let mut q = ask::Question::new(
15366            run.to_owned(),
15367            "implement".to_owned(),
15368            "impl-A".to_owned(),
15369            "Which storage backend should the cache use?".to_owned(),
15370            String::new(),
15371            vec!["SQLite".to_owned(), "Redis".to_owned()],
15372        );
15373        store.put(&mut q).unwrap();
15374        q
15375    }
15376
15377    #[test]
15378    fn a_failed_runs_open_question_is_abandoned() {
15379        ask_test_home();
15380        let store = ask::Questions::open();
15381        let mut runner = runner_at(RunStatus::Failed);
15382        let run = runner.state.id.clone();
15383        let q = ask_open_question(&store, &run);
15384
15385        runner.settle_questions();
15386
15387        let back = store.get(&q.id).unwrap();
15388        assert!(
15389            !back.status.open(),
15390            "the seat that asked died with the run; nobody is left to read an answer"
15391        );
15392        assert!(
15393            back.detail.contains(&run) && back.detail.contains("failed"),
15394            "the reason names what the run became, not just that it is gone: {}",
15395            back.detail
15396        );
15397    }
15398
15399    #[test]
15400    fn a_merged_runs_open_question_is_abandoned_too() {
15401        ask_test_home();
15402        let store = ask::Questions::open();
15403        // A run that finishes cleanly still leaves nobody to read an answer -
15404        // this is not only a failure-path cleanup.
15405        for status in [RunStatus::Merged, RunStatus::Ready] {
15406            let mut runner = runner_at(status);
15407            let run = runner.state.id.clone();
15408            let q = ask_open_question(&store, &run);
15409
15410            runner.settle_questions();
15411
15412            let back = store.get(&q.id).unwrap();
15413            assert!(
15414                !back.status.open(),
15415                "{status:?} run's question must not outlive the run"
15416            );
15417        }
15418    }
15419
15420    #[test]
15421    fn a_still_resumable_runs_open_question_is_left_alone() {
15422        ask_test_home();
15423        let store = ask::Questions::open();
15424        // `Blocked` and `Stalled` can still be resumed — the candidates, the
15425        // review round and the seat sessions are all still on disk — so a
15426        // question asked mid-round may yet get a real answer from a real
15427        // resume. Sweeping it here would be exactly the failure mode this
15428        // whole feature exists to avoid on the other side.
15429        for status in [RunStatus::Blocked, RunStatus::Stalled] {
15430            let mut runner = runner_at(status);
15431            let run = runner.state.id.clone();
15432            let q = ask_open_question(&store, &run);
15433
15434            runner.settle_questions();
15435
15436            let back = store.get(&q.id).unwrap();
15437            assert!(
15438                back.status.open(),
15439                "{status:?} is still alive; the question must still be waiting"
15440            );
15441        }
15442    }
15443
15444    #[test]
15445    fn settle_questions_never_touches_an_already_answered_question() {
15446        ask_test_home();
15447        let store = ask::Questions::open();
15448        let mut runner = runner_at(RunStatus::Failed);
15449        let run = runner.state.id.clone();
15450        let mut q = ask_open_question(&store, &run);
15451        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15452            .unwrap();
15453        store.put(&mut q).unwrap();
15454
15455        // Called twice, the way a crash-recovered daemon reclaim and the
15456        // graph's own cleanup both can for the same run — `abandon_for_run`
15457        // only ever touches what is still open, so this must be inert both
15458        // times, not merely the second.
15459        runner.settle_questions();
15460        runner.settle_questions();
15461
15462        let back = store.get(&q.id).unwrap();
15463        assert_eq!(
15464            back.status,
15465            ask::QuestionStatus::Answered,
15466            "a real answer is a decision on record, never overwritten by a sweep"
15467        );
15468    }
15469
15470    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
15471    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
15472    /// without merging, whose winner is still the operator's answer to read.
15473    /// Nothing previously called `fold_run` itself with a real `tally`, so
15474    /// this is the first test to pin down the one distinction the whole
15475    /// automatic-fold feature depends on: the winner's worktree and branch
15476    /// must survive, everything else sharing the run's worktree bay - a
15477    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
15478    /// second sweep treats every non-winner directory under the bay alike -
15479    /// must not.
15480    #[tokio::test]
15481    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15482        crate::run::pin_test_home();
15483        let tmp = tempfile::tempdir().expect("tempdir");
15484        let repo = tmp.path().join("repo");
15485        std::fs::create_dir_all(&repo).unwrap();
15486        init_repo(&repo);
15487
15488        let mut config = Config::default();
15489        config.graph.worktree_root = Some(tmp.path().join("wt"));
15490
15491        let mut state = RunState::new(
15492            repo.clone(),
15493            "main".to_owned(),
15494            "deadbeef".to_owned(),
15495            "task".to_owned(),
15496            config,
15497        );
15498        let root = state.worktree_root();
15499        let wt_a = root.join("cand-A");
15500        let wt_b = root.join("cand-B");
15501        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15502            .await
15503            .expect("worktree A");
15504        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15505            .await
15506            .expect("worktree B");
15507
15508        state.candidates = vec![
15509            Candidate {
15510                index: 0,
15511                label: 'A',
15512                agent: "alpha".to_owned(),
15513                branch: "magi/x/A".to_owned(),
15514                worktree: wt_a.clone(),
15515                summary: String::new(),
15516                stat: String::new(),
15517                files: 0,
15518                commits: 0,
15519                empty: false,
15520                failed: None,
15521                verified_noop: None,
15522                duration_ms: 0,
15523                folded: false,
15524            },
15525            Candidate {
15526                index: 1,
15527                label: 'B',
15528                agent: "beta".to_owned(),
15529                branch: "magi/x/B".to_owned(),
15530                worktree: wt_b.clone(),
15531                summary: String::new(),
15532                stat: String::new(),
15533                files: 0,
15534                commits: 0,
15535                empty: false,
15536                failed: None,
15537                verified_noop: None,
15538                duration_ms: 0,
15539                folded: false,
15540            },
15541        ];
15542        state.tally = Some(Tally {
15543            first_choice: BTreeMap::from([('A', 1)]),
15544            borda: BTreeMap::new(),
15545            winner: 'A',
15546            rankings: 1,
15547            unanimous_initial: true,
15548            deliberated: false,
15549            changed_votes: 0,
15550            unanimous_final: true,
15551            tie_break: None,
15552            judges: 1,
15553            present: 1,
15554            quorum: 1,
15555            met_quorum: true,
15556            uncontested: None,
15557        });
15558        state.status = RunStatus::Ready;
15559
15560        fold_run(&mut state, false, &crate::run::home())
15561            .await
15562            .expect("fold_run");
15563
15564        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15565        assert!(
15566            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15567            "the unmerged winner's branch survives"
15568        );
15569        assert!(
15570            !state.candidates[0].folded,
15571            "the winner is not marked folded"
15572        );
15573
15574        assert!(!wt_b.exists(), "the loser's worktree is removed");
15575        assert!(
15576            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15577            "the loser's branch is removed"
15578        );
15579        assert!(state.candidates[1].folded, "the loser is marked folded");
15580    }
15581
15582    #[tokio::test]
15583    #[cfg(unix)]
15584    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15585        use std::os::fd::AsRawFd as _;
15586        use std::time::Duration;
15587
15588        let tmp = tempfile::tempdir().unwrap();
15589        let repo = tmp.path().join("repo");
15590        std::fs::create_dir_all(&repo).unwrap();
15591        init_repo(&repo);
15592        let common = repo.join(".git");
15593        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15594        let home = tmp.path().join("home");
15595        let mut config = Config::default();
15596        config.coordination = crate::config::test_coordination();
15597        config.graph.worktree_root = Some(tmp.path().join("wt"));
15598        config.graph.timeout_resource_wait = 1;
15599        let mut state = RunState::new(
15600            repo.clone(),
15601            "main".into(),
15602            "deadbeef".into(),
15603            "task".into(),
15604            config,
15605        );
15606        let root = state.worktree_root();
15607        let candidate = root.join("cand-A");
15608        let reviewer = root.join("review-1");
15609        let branch = state.branch_for('A');
15610        git::worktree_add_branch(&repo, &candidate, &branch, "main")
15611            .await
15612            .unwrap();
15613        git::worktree_add_detached(&repo, &reviewer, "main")
15614            .await
15615            .unwrap();
15616        git::acquire_worktree_config(&repo).await.unwrap();
15617        state.enabled_worktree_config = true;
15618        state.status = RunStatus::Ready;
15619        state.candidates.push(Candidate {
15620            index: 0,
15621            label: 'A',
15622            agent: "implement".into(),
15623            branch: branch.clone(),
15624            worktree: candidate.clone(),
15625            summary: String::new(),
15626            stat: String::new(),
15627            files: 0,
15628            commits: 0,
15629            empty: false,
15630            failed: None,
15631            verified_noop: None,
15632            duration_ms: 0,
15633            folded: false,
15634        });
15635        state.save_under(&home).unwrap();
15636        let config_before = std::fs::read(common.join("config")).unwrap();
15637        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15638            .await
15639            .unwrap();
15640        // A separate OS handle bypasses the launcher's local gate, exactly as
15641        // an external updater does, and must still prevent all folding work.
15642        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15643        assert_eq!(
15644            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15645            0
15646        );
15647        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15648        assert!(
15649            tokio::time::timeout(Duration::from_millis(200), &mut fold)
15650                .await
15651                .is_err()
15652        );
15653        assert!(candidate.exists());
15654        assert!(reviewer.exists());
15655        assert!(git::branch_exists(&repo, &branch).await.unwrap());
15656        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15657        assert_eq!(
15658            git::git(&repo, &["worktree", "list", "--porcelain"])
15659                .await
15660                .unwrap(),
15661            worktrees_before
15662        );
15663        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15664            .await
15665            .unwrap()
15666            .unwrap_err();
15667        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15668        drop(fold);
15669        assert!(candidate.exists() && reviewer.exists());
15670        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15671        assert_eq!(
15672            state.resource_block.as_ref().unwrap().kind,
15673            "repository coordination"
15674        );
15675        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15676        drop(external);
15677        let mut fold = Box::pin(fold_run(&mut state, true, &home));
15678        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15679            .await
15680            .unwrap()
15681            .unwrap();
15682        drop(fold);
15683        assert!(removed.contains(&branch));
15684        assert!(!root.exists());
15685        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15686        assert!(state.candidates[0].folded);
15687        assert!(!state.enabled_worktree_config);
15688        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15689        let enabled = git::git_raw(
15690            &repo,
15691            &["config", "--local", "--get", "extensions.worktreeConfig"],
15692        )
15693        .await
15694        .unwrap();
15695        assert!(!enabled.ok(), "the last config reference was released");
15696    }
15697
15698    /// A branch handed to a later run is that run's (and its pull request's):
15699    /// folding the run that released it must not delete it.
15700    #[tokio::test]
15701    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15702        let tmp = tempfile::tempdir().expect("tempdir");
15703        let repo = tmp.path().join("repo");
15704        std::fs::create_dir_all(&repo).unwrap();
15705        init_repo(&repo);
15706        let home = tmp.path().join("home");
15707
15708        let mut config = Config::default();
15709        config.graph.worktree_root = Some(tmp.path().join("wt"));
15710        let mut state = RunState::new(
15711            repo.clone(),
15712            "main".to_owned(),
15713            "deadbeef".to_owned(),
15714            "task".to_owned(),
15715            config,
15716        );
15717        // The worktree is already gone (released); the branch survives.
15718        git::git(&repo, &["branch", "magi/x/A", "main"])
15719            .await
15720            .expect("branch");
15721        state.candidates = vec![Candidate {
15722            index: 0,
15723            label: 'A',
15724            agent: "alpha".to_owned(),
15725            branch: "magi/x/A".to_owned(),
15726            worktree: state.worktree_root().join("cand-A"),
15727            summary: String::new(),
15728            stat: String::new(),
15729            files: 0,
15730            commits: 0,
15731            empty: false,
15732            failed: None,
15733            verified_noop: None,
15734            duration_ms: 0,
15735            folded: true,
15736        }];
15737        state.released_to = Some("20260901-000000-new1".to_owned());
15738        state.released_branches = vec!["magi/x/A".to_owned()];
15739
15740        fold_run(&mut state, true, &home).await.expect("fold_run");
15741        state.status = RunStatus::Merged;
15742        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15743        state.merge = Some(crate::run::MergeOutcome {
15744            mode: MergeMode::Pr,
15745            ok: true,
15746            detail: "confirmed merged".into(),
15747            empty: false,
15748        });
15749        fold_run(&mut state, true, &home)
15750            .await
15751            .expect("merged fold");
15752
15753        assert!(
15754            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15755            "the handed-over branch survives a fold"
15756        );
15757    }
15758
15759    /// A winner with nothing ahead of the base is caught before `gh` is ever
15760    /// asked for a pull request, and the message carries what the task's
15761    /// references resolved to.
15762    #[tokio::test]
15763    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15764        let tmp = tempfile::tempdir().expect("tempdir");
15765        let repo = tmp.path().join("repo");
15766        std::fs::create_dir_all(&repo).unwrap();
15767        init_repo(&repo);
15768        let run = |args: &[&str]| {
15769            let out = std::process::Command::new("git")
15770                .quiet()
15771                .args(args)
15772                .current_dir(&repo)
15773                .output()
15774                .expect("spawn git");
15775            assert!(out.status.success(), "git {args:?}");
15776        };
15777        run(&["branch", "magi/x/A"]);
15778        run(&["checkout", "-q", "-b", "magi/x/B"]);
15779        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15780        run(&["add", "-A"]);
15781        run(&["commit", "-q", "-m", "work"]);
15782        run(&["checkout", "-q", "main"]);
15783
15784        // A pull request is compared against the remote's base, so the
15785        // fixture needs one. Before it exists nothing can be read, and the
15786        // answer must be "not empty".
15787        let probe = RunState::new(
15788            repo.clone(),
15789            "main".to_owned(),
15790            "deadbeef".to_owned(),
15791            "task".to_owned(),
15792            Config::default(),
15793        );
15794        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15795        let bare = tmp.path().join("origin.git");
15796        let out = std::process::Command::new("git")
15797            .quiet()
15798            .args(["init", "-q", "--bare"])
15799            .arg(&bare)
15800            .output()
15801            .expect("spawn git");
15802        assert!(out.status.success(), "git init --bare");
15803        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15804        run(&["push", "-q", "origin", "main"]);
15805
15806        let mut state = RunState::new(
15807            repo.clone(),
15808            "main".to_owned(),
15809            "deadbeef".to_owned(),
15810            "task".to_owned(),
15811            Config::default(),
15812        );
15813        state.seeds = vec![refs::Seed {
15814            token: "magi/27b2/A".to_owned(),
15815            kind: refs::SeedKind::Unresolved,
15816            sha: String::new(),
15817            branch: true,
15818            detail: "no branch or commit named magi/27b2/A".to_owned(),
15819        }];
15820
15821        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15822        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15823        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15824        let detail = empty_candidate_detail(&state, "main");
15825        assert!(detail.starts_with("empty candidate"), "{detail}");
15826        assert!(!detail.contains("magi/27b2/A"), "{detail}");
15827        assert!(detail.contains("--start-from"), "{detail}");
15828    }
15829
15830    /// `status == Ready` used to be read as "this is the harmless
15831    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
15832    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
15833    /// run's PR was closed without merging — and reentering `merge` with
15834    /// `mode` still `Pr` does not know the difference, so it pushed and
15835    /// opened a second pull request. `mode == Local` reproduces the same
15836    /// blind spot without a network call: reentry must not attempt another
15837    /// git merge once this node has already recorded an outcome.
15838    #[tokio::test]
15839    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15840        let tmp = tempfile::tempdir().expect("tempdir");
15841        let repo = tmp.path().join("repo");
15842        std::fs::create_dir_all(&repo).unwrap();
15843        init_repo(&repo);
15844
15845        let mut config = Config::default();
15846        config.merge.mode = MergeMode::Local;
15847
15848        let mut state = RunState::new(
15849            repo.clone(),
15850            "main".to_owned(),
15851            "deadbeef".to_owned(),
15852            "task".to_owned(),
15853            config,
15854        );
15855        state.candidates = vec![Candidate {
15856            index: 0,
15857            label: 'A',
15858            agent: "alpha".to_owned(),
15859            branch: "does-not-exist".to_owned(),
15860            worktree: repo.clone(),
15861            summary: String::new(),
15862            stat: String::new(),
15863            files: 0,
15864            commits: 0,
15865            empty: false,
15866            failed: None,
15867            verified_noop: None,
15868            duration_ms: 0,
15869            folded: false,
15870        }];
15871        state.tally = Some(Tally {
15872            first_choice: BTreeMap::from([('A', 1)]),
15873            borda: BTreeMap::new(),
15874            winner: 'A',
15875            rankings: 1,
15876            unanimous_initial: true,
15877            deliberated: false,
15878            changed_votes: 0,
15879            unanimous_final: true,
15880            tie_break: None,
15881            judges: 0,
15882            present: 0,
15883            quorum: 0,
15884            met_quorum: true,
15885            uncontested: Some("only candidate A produced a change".to_owned()),
15886        });
15887        state.reviews = vec![ReviewRound {
15888            round: 1,
15889            head: "deadbeef".to_owned(),
15890            verified_head: None,
15891            verified_at: None,
15892            reviews: Vec::new(),
15893            e2e: Vec::new(),
15894            fix: None,
15895            blocking: 0,
15896            answered: 0,
15897            expected: 0,
15898            clean: true,
15899            verify_retried: false,
15900            e2e_deferred: false,
15901            e2e_defer_reason: None,
15902            progressed: false,
15903            vote_split: false,
15904            reconsideration: Vec::new(),
15905            verdict: None,
15906        }];
15907        state.gate = vec![CommandOutcome {
15908            command: "test".to_owned(),
15909            code: Some(0),
15910            output_tail: String::new(),
15911            duration_ms: 0,
15912            resource_blocked: false,
15913        }];
15914        state.gate_ran = true;
15915        // Reached its conclusion already — e.g. `land` closing the PR without
15916        // merging it, which (like the honest `MergeMode::None` path) leaves
15917        // `status` at `Ready`. The recorded outcome is what actually marks
15918        // this node done.
15919        state.status = RunStatus::Ready;
15920        state.merge = Some(MergeOutcome {
15921            mode: MergeMode::Local,
15922            ok: false,
15923            detail: "already concluded".to_owned(),
15924            empty: false,
15925        });
15926
15927        let mut runner = Runner {
15928            state,
15929            roles: ResolvedRoles {
15930                implementers: Vec::new(),
15931                judges: Vec::new(),
15932                reviewers: Vec::new(),
15933                fixer: None,
15934                conductor: conductor(),
15935                implementer_roster: Vec::new(),
15936                judge_roster: Vec::new(),
15937                reviewer_roster: Vec::new(),
15938            },
15939            sem: Arc::new(Semaphore::new(1)),
15940            pause: Pause::new(),
15941            interrupt: Pause::new(),
15942        };
15943
15944        runner.merge().await.expect("merge");
15945
15946        assert_eq!(
15947            runner.state.status,
15948            RunStatus::Ready,
15949            "a concluded run's status must not change on reentry"
15950        );
15951        assert_eq!(
15952            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15953            Some("already concluded"),
15954            "merge must not run again once the node already recorded an outcome"
15955        );
15956    }
15957
15958    /// `gate` leaves `state.gate_ran` false both before it has ever run and
15959    /// when its last attempt was resource-blocked (the shared build cache
15960    /// could not be acquired or confirmed fresh in time - see
15961    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
15962    /// `Vec` this also leaves behind used to read as "nothing failed" and let
15963    /// a run merge a tree the gate never actually checked - exactly the case
15964    /// a contended cache produces on every retry until it clears. `merge`
15965    /// must refuse until `gate` has actually recorded an attempt.
15966    #[tokio::test]
15967    async fn merge_refuses_a_gate_that_has_not_actually_run() {
15968        let tmp = tempfile::tempdir().expect("tempdir");
15969        let repo = tmp.path().join("repo");
15970        std::fs::create_dir_all(&repo).unwrap();
15971        init_repo(&repo);
15972
15973        let mut config = Config::default();
15974        config.merge.mode = MergeMode::Local;
15975
15976        let mut state = RunState::new(
15977            repo.clone(),
15978            "main".to_owned(),
15979            "deadbeef".to_owned(),
15980            "task".to_owned(),
15981            config,
15982        );
15983        state.candidates = vec![Candidate {
15984            index: 0,
15985            label: 'A',
15986            agent: "alpha".to_owned(),
15987            branch: "does-not-exist".to_owned(),
15988            worktree: repo.clone(),
15989            summary: String::new(),
15990            stat: String::new(),
15991            files: 0,
15992            commits: 0,
15993            empty: false,
15994            failed: None,
15995            verified_noop: None,
15996            duration_ms: 0,
15997            folded: false,
15998        }];
15999        state.tally = Some(Tally {
16000            first_choice: BTreeMap::from([('A', 1)]),
16001            borda: BTreeMap::new(),
16002            winner: 'A',
16003            rankings: 1,
16004            unanimous_initial: true,
16005            deliberated: false,
16006            changed_votes: 0,
16007            unanimous_final: true,
16008            tie_break: None,
16009            judges: 0,
16010            present: 0,
16011            quorum: 0,
16012            met_quorum: true,
16013            uncontested: Some("only candidate A produced a change".to_owned()),
16014        });
16015        state.reviews = vec![ReviewRound {
16016            round: 1,
16017            head: "deadbeef".to_owned(),
16018            verified_head: None,
16019            verified_at: None,
16020            reviews: Vec::new(),
16021            e2e: Vec::new(),
16022            fix: None,
16023            blocking: 0,
16024            answered: 0,
16025            expected: 0,
16026            clean: true,
16027            verify_retried: false,
16028            e2e_deferred: false,
16029            e2e_defer_reason: None,
16030            progressed: false,
16031            vote_split: false,
16032            reconsideration: Vec::new(),
16033            verdict: None,
16034        }];
16035        // The point: `gate` has not recorded anything yet.
16036        state.gate = Vec::new();
16037        state.gate_ran = false;
16038        state.status = RunStatus::Gating;
16039
16040        let mut runner = Runner {
16041            state,
16042            roles: ResolvedRoles {
16043                implementers: Vec::new(),
16044                judges: Vec::new(),
16045                reviewers: Vec::new(),
16046                fixer: None,
16047                conductor: conductor(),
16048                implementer_roster: Vec::new(),
16049                judge_roster: Vec::new(),
16050                reviewer_roster: Vec::new(),
16051            },
16052            sem: Arc::new(Semaphore::new(1)),
16053            pause: Pause::new(),
16054            interrupt: Pause::new(),
16055        };
16056
16057        runner.merge().await.expect("merge");
16058
16059        assert!(
16060            runner.state.merge.is_none(),
16061            "an empty gate must never be read as a passing one: {:?}",
16062            runner.state.merge
16063        );
16064    }
16065
16066    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
16067    /// commands configured and `merge.mode` is `none` (a review-only run).
16068    /// `gate` must still record a real attempt — zero commands, vacuously
16069    /// passed — rather than leaving `state.gate` empty in a way `merge`
16070    /// cannot tell apart from "never ran"; otherwise the run reaches
16071    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
16072    #[tokio::test]
16073    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16074        let tmp = tempfile::tempdir().expect("tempdir");
16075        let repo = tmp.path().join("repo");
16076        std::fs::create_dir_all(&repo).unwrap();
16077        init_repo(&repo);
16078
16079        // Default config: `verify.gate` empty, `merge.mode` is `none`.
16080        let config = Config::default();
16081
16082        let mut state = RunState::new(
16083            repo.clone(),
16084            "main".to_owned(),
16085            "deadbeef".to_owned(),
16086            "task".to_owned(),
16087            config,
16088        );
16089        state.candidates = vec![Candidate {
16090            index: 0,
16091            label: 'A',
16092            agent: "alpha".to_owned(),
16093            branch: "does-not-exist".to_owned(),
16094            worktree: repo.clone(),
16095            summary: String::new(),
16096            stat: String::new(),
16097            files: 0,
16098            commits: 0,
16099            empty: false,
16100            failed: None,
16101            verified_noop: None,
16102            duration_ms: 0,
16103            folded: false,
16104        }];
16105        state.tally = Some(Tally {
16106            first_choice: BTreeMap::from([('A', 1)]),
16107            borda: BTreeMap::new(),
16108            winner: 'A',
16109            rankings: 1,
16110            unanimous_initial: true,
16111            deliberated: false,
16112            changed_votes: 0,
16113            unanimous_final: true,
16114            tie_break: None,
16115            judges: 0,
16116            present: 0,
16117            quorum: 0,
16118            met_quorum: true,
16119            uncontested: Some("only candidate A produced a change".to_owned()),
16120        });
16121        state.reviews = vec![ReviewRound {
16122            round: 1,
16123            head: "deadbeef".to_owned(),
16124            verified_head: None,
16125            verified_at: None,
16126            reviews: Vec::new(),
16127            e2e: Vec::new(),
16128            fix: None,
16129            blocking: 0,
16130            answered: 0,
16131            expected: 0,
16132            clean: true,
16133            verify_retried: false,
16134            e2e_deferred: false,
16135            e2e_defer_reason: None,
16136            progressed: false,
16137            vote_split: false,
16138            reconsideration: Vec::new(),
16139            verdict: None,
16140        }];
16141
16142        let mut runner = Runner {
16143            state,
16144            roles: ResolvedRoles {
16145                implementers: Vec::new(),
16146                judges: Vec::new(),
16147                reviewers: Vec::new(),
16148                fixer: None,
16149                conductor: conductor(),
16150                implementer_roster: Vec::new(),
16151                judge_roster: Vec::new(),
16152                reviewer_roster: Vec::new(),
16153            },
16154            sem: Arc::new(Semaphore::new(1)),
16155            pause: Pause::new(),
16156            interrupt: Pause::new(),
16157        };
16158
16159        runner.gate().await.expect("gate");
16160        assert!(
16161            runner.state.gate_ran,
16162            "zero configured commands is still a real attempt, not an unrun gate"
16163        );
16164        assert!(runner.state.gate.is_empty());
16165        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16166        assert_ne!(
16167            runner.state.status,
16168            RunStatus::Blocked,
16169            "a gate with nothing to check must not read as failed"
16170        );
16171
16172        runner.merge().await.expect("merge");
16173        assert_eq!(
16174            runner.state.status,
16175            RunStatus::Ready,
16176            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16177        );
16178    }
16179
16180    /// `Config::cache_dir` is derived from `verify.e2e` as well as
16181    /// `verify.gate` (so the e2e leg and the final gate never build against
16182    /// different directories). With zero `verify.gate` commands but a
16183    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
16184    /// that lease before discovering it had nothing to run - so a repo with
16185    /// no gate commands could come back `resource_blocked` (and therefore
16186    /// still `gate_ran == false`) on nothing but an unrelated run holding the
16187    /// cache, exactly the contention this run's own zero commands could
16188    /// never have touched. `gate` must recognise there is nothing to check
16189    /// before it ever asks for the lease.
16190    #[tokio::test]
16191    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16192        crate::run::pin_test_home();
16193        let home = crate::run::home();
16194
16195        let tmp = tempfile::tempdir().expect("tempdir");
16196        let repo = tmp.path().join("repo");
16197        std::fs::create_dir_all(&repo).unwrap();
16198        init_repo(&repo);
16199        // Unique to this test, so holding its lease cannot collide with
16200        // another test sharing the same process-wide `home`.
16201        let cache_dir = tmp.path().join("target");
16202
16203        let mut config = Config::default();
16204        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16205        // `verify.gate` stays empty (the default). Bounded so a regression
16206        // that does start waiting fails the test in seconds, not hangs it.
16207        config.graph.timeout_verify = Some(2);
16208        config.graph.timeout_resource_wait = 2;
16209
16210        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16211        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16212            .expect("no io error acquiring directly")
16213        {
16214            crate::cache::AcquireOutcome::Acquired(g) => g,
16215            crate::cache::AcquireOutcome::Busy(b) => {
16216                panic!("expected the direct acquire to win the lease first: {b:?}")
16217            }
16218        };
16219
16220        let mut state = RunState::new(
16221            repo.clone(),
16222            "main".to_owned(),
16223            "deadbeef".to_owned(),
16224            "task".to_owned(),
16225            config,
16226        );
16227        state.candidates = vec![Candidate {
16228            index: 0,
16229            label: 'A',
16230            agent: "alpha".to_owned(),
16231            branch: "does-not-exist".to_owned(),
16232            worktree: repo.clone(),
16233            summary: String::new(),
16234            stat: String::new(),
16235            files: 0,
16236            commits: 0,
16237            empty: false,
16238            failed: None,
16239            verified_noop: None,
16240            duration_ms: 0,
16241            folded: false,
16242        }];
16243        state.tally = Some(Tally {
16244            first_choice: BTreeMap::from([('A', 1)]),
16245            borda: BTreeMap::new(),
16246            winner: 'A',
16247            rankings: 1,
16248            unanimous_initial: true,
16249            deliberated: false,
16250            changed_votes: 0,
16251            unanimous_final: true,
16252            tie_break: None,
16253            judges: 0,
16254            present: 0,
16255            quorum: 0,
16256            met_quorum: true,
16257            uncontested: Some("only candidate A produced a change".to_owned()),
16258        });
16259        state.reviews = vec![ReviewRound {
16260            round: 1,
16261            head: "deadbeef".to_owned(),
16262            verified_head: None,
16263            verified_at: None,
16264            reviews: Vec::new(),
16265            e2e: Vec::new(),
16266            fix: None,
16267            blocking: 0,
16268            answered: 0,
16269            expected: 0,
16270            clean: true,
16271            verify_retried: false,
16272            e2e_deferred: false,
16273            e2e_defer_reason: None,
16274            progressed: false,
16275            vote_split: false,
16276            reconsideration: Vec::new(),
16277            verdict: None,
16278        }];
16279
16280        let mut runner = Runner {
16281            state,
16282            roles: ResolvedRoles {
16283                implementers: Vec::new(),
16284                judges: Vec::new(),
16285                reviewers: Vec::new(),
16286                fixer: None,
16287                conductor: conductor(),
16288                implementer_roster: Vec::new(),
16289                judge_roster: Vec::new(),
16290                reviewer_roster: Vec::new(),
16291            },
16292            sem: Arc::new(Semaphore::new(1)),
16293            pause: Pause::new(),
16294            interrupt: Pause::new(),
16295        };
16296
16297        let started = std::time::Instant::now();
16298        runner.gate().await.expect("gate");
16299        assert!(
16300            started.elapsed() < Duration::from_secs(1),
16301            "a gate with nothing to run must never wait on a lease it never needed"
16302        );
16303        assert!(
16304            runner.state.gate_ran,
16305            "zero commands is still a real, immediate attempt"
16306        );
16307        assert!(runner.state.gate.is_empty());
16308        assert_ne!(
16309            runner.state.status,
16310            RunStatus::Blocked,
16311            "must not read as resource-blocked on a lease it never asked for"
16312        );
16313    }
16314
16315    /// A separate process carries a live worker's environment throughout all
16316    /// verification entry points; the parallel harness never changes its env.
16317    #[cfg(unix)]
16318    #[tokio::test]
16319    async fn verification_is_standalone_inside_a_live_worker() {
16320        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
16321        if std::env::var_os(CHILD).is_none() {
16322            let home = tempfile::tempdir().unwrap();
16323            let generation = crate::worker::pin_current(home.path()).unwrap();
16324            let output = std::process::Command::new(std::env::current_exe().unwrap())
16325                .args([
16326                    "--exact",
16327                    "graph::tests::verification_is_standalone_inside_a_live_worker",
16328                    "--nocapture",
16329                ])
16330                .env(CHILD, "1")
16331                .env("MAGI_HOME", home.path())
16332                .env("MAGI_WORKER_HOME", home.path())
16333                .env("MAGI_WORKER_ID", "abcdef")
16334                .env("MAGI_WORKER_GENERATION", &generation.id)
16335                .env("MAGI_WORKER_EXE", &generation.executable)
16336                .env("MAGI_RUN", "verification-parent")
16337                .env("MAGI_NODE", "gate")
16338                .env("MAGI_SEAT", "worker-fixture")
16339                .env("MAGI_TURN_SCOPE", "fixture-scope")
16340                .env("MAGI_PROCESS_OWNER", "fixture-owner")
16341                .env("MAGI_NO_AUTOUPDATE", "0")
16342                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
16343                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
16344                .env(
16345                    "VERIFY_EXPECT_TOOLCHAIN",
16346                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
16347                )
16348                .env(
16349                    "VERIFY_EXPECT_CACHE",
16350                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
16351                )
16352                .output()
16353                .unwrap();
16354            assert!(
16355                output.status.success()
16356                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16357                "{}\n{}",
16358                String::from_utf8_lossy(&output.stdout),
16359                String::from_utf8_lossy(&output.stderr)
16360            );
16361            return;
16362        }
16363        let home = crate::run::home();
16364        let directory = home.join("workers/jobs/abcdef");
16365        std::fs::create_dir_all(&directory).unwrap();
16366        let job = crate::worker::Job {
16367            id: "abcdef".into(),
16368            sequence: 1,
16369            kind: "task".into(),
16370            key: "task:verification".into(),
16371            payload: serde_json::json!({}),
16372            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
16373            version: Some(env!("CARGO_PKG_VERSION").into()),
16374            status: crate::worker::Status::Running,
16375            result: None,
16376            pid: Some(std::process::id()),
16377            created_at: 0,
16378            finished_at: None,
16379        };
16380        std::fs::write(
16381            directory.join("job.json"),
16382            serde_json::to_vec(&job).unwrap(),
16383        )
16384        .unwrap();
16385        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
16386        let assert_worker = || {
16387            assert_eq!(
16388                crate::worker::current_job(&home).unwrap().unwrap().id,
16389                job.id
16390            );
16391            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
16392            assert_eq!(
16393                crate::worker::helper_executable(&home).unwrap(),
16394                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
16395            );
16396            assert!(
16397                crate::updater::Checker::new(&crate::config::Update {
16398                    mode: crate::config::UpdateMode::Notify,
16399                    interval: None,
16400                    auto_restart: false,
16401                })
16402                .is_none(),
16403                "the driver remains pinned, even while tests are standalone"
16404            );
16405        };
16406        assert_worker();
16407        let repo = home.join("repo");
16408        std::fs::create_dir_all(&repo).unwrap();
16409        init_repo(&repo);
16410        // Keep storage, seat/control context, containment and tool lookup.
16411        // No inherited values (or credentials) are printed on success.
16412        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
16413            test -z "${MAGI_WORKER_GENERATION+x}" &&
16414            test -z "${MAGI_WORKER_HOME+x}" &&
16415            test -z "${MAGI_WORKER_EXE+x}" &&
16416            test "$MAGI_HOME/repo" -ef . &&
16417            test "$MAGI_RUN" = verification-parent &&
16418            test "$MAGI_NODE" = gate &&
16419            test "$MAGI_SEAT" = worker-fixture &&
16420            test "$MAGI_TURN_SCOPE" = fixture-scope &&
16421            test -n "$MAGI_PROCESS_OWNER" &&
16422            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
16423            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
16424            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
16425            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
16426        // Negative control: an ordinary child inherits the contamination.
16427        assert!(
16428            !std::process::Command::new("sh")
16429                .args(["-c", probe])
16430                .current_dir(&repo)
16431                .status()
16432                .unwrap()
16433                .success()
16434        );
16435        let unrelated_home = tempfile::tempdir().unwrap();
16436        assert!(
16437            crate::worker::current_job(unrelated_home.path()).is_err(),
16438            "an inherited worker id cannot be resolved in a unit fixture's home"
16439        );
16440        let mut runner = runner_at(RunStatus::Gating);
16441        runner.state.repo = repo.clone();
16442        runner.state.config.disk.min_free_bytes = 0;
16443        runner.state.config.coordination = crate::config::test_coordination();
16444        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16445        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
16446        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
16447        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
16448        let winner = Candidate {
16449            index: 0,
16450            label: 'A',
16451            agent: "fixture".into(),
16452            branch: "main".into(),
16453            worktree: repo.clone(),
16454            summary: String::new(),
16455            stat: String::new(),
16456            files: 0,
16457            commits: 0,
16458            empty: false,
16459            failed: None,
16460            verified_noop: None,
16461            duration_ms: 0,
16462            folded: false,
16463        };
16464        let (e2e, retried, pids) = run_e2e_with_retry(
16465            &mut runner.state,
16466            &["sh".into(), "-c".into()],
16467            &[format!("{probe} && printf initial")],
16468            &repo,
16469            Duration::from_secs(10),
16470            "initial",
16471        )
16472        .await;
16473        assert!(!retried && pids.is_empty());
16474        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
16475        assert_eq!(e2e[0].output_tail, "initial");
16476        let gate = runner.run_gate(&winner).await.unwrap();
16477        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
16478        // An interrupted repair must recheck e2e from saved state. Landing
16479        // reuses that state and must run both pre-gate and gate again.
16480        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
16481            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
16482            passed: false,
16483            outcomes: Vec::new(),
16484        });
16485        runner.state.save().unwrap();
16486        let mut resumed = RunState::load(&runner.state.id).unwrap();
16487        let wait = crate::repo_lock::WaitPolicy::default();
16488        assert!(matches!(
16489            check_gate_repair_e2e(&mut resumed, &winner, &wait)
16490                .await
16491                .unwrap(),
16492            GateFix::Retry
16493        ));
16494        let saved = RunState::load(&resumed.id).unwrap();
16495        let repair = saved.gate_repair_e2e.unwrap();
16496        assert!(repair.passed);
16497        assert_eq!(repair.outcomes.len(), 1);
16498        assert!(repair.outcomes[0].ok());
16499        let landing = run_landing_gate(&mut resumed, &winner, &wait)
16500            .await
16501            .unwrap();
16502        assert_eq!(landing.len(), 1);
16503        assert!(landing[0].ok(), "{landing:?}");
16504        assert_eq!(
16505            std::fs::read_to_string(repo.join("phases")).unwrap(),
16506            "pre\ngate\ne2e\npre\ngate\n"
16507        );
16508        assert_worker();
16509    }
16510
16511    #[cfg(unix)]
16512    #[tokio::test]
16513    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16514        crate::run::pin_test_home();
16515        let tmp = tempfile::tempdir().unwrap();
16516        let mut config = Config::default();
16517        config.disk.min_free_bytes = 0;
16518        let mut state = RunState::new(
16519            tmp.path().to_owned(),
16520            "main".into(),
16521            "base".into(),
16522            "verification evidence".into(),
16523            config,
16524        );
16525        let (outcomes, pids) = run_commands(
16526            &mut state,
16527            "gate",
16528            "gate",
16529            1,
16530            &["sh".into(), "-c".into()],
16531            &[
16532                "printf before-timeout; exec sleep 30".into(),
16533                "kill -TERM $$".into(),
16534                "exit 7".into(),
16535                "printf success".into(),
16536            ],
16537            tmp.path(),
16538            Duration::from_secs(1),
16539        )
16540        .await;
16541        wait_for_timed_out_children_to_die(&pids).await;
16542        assert_eq!(
16543            pids.len(),
16544            1,
16545            "unleased timeout retains its death-check PID"
16546        );
16547        assert_eq!(outcomes.len(), 4);
16548        assert_eq!(outcomes[0].code, None);
16549        assert!(outcomes[0].output_tail.contains("before-timeout"));
16550        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16551        assert_eq!(outcomes[1].code, None);
16552        assert!(outcomes[1].output_tail.contains("command terminated:"));
16553        assert!(!outcomes[1].output_tail.contains("timed out"));
16554        assert_eq!(outcomes[2].code, Some(7));
16555        assert!(!outcomes[2].output_tail.contains("timed out"));
16556        assert_eq!(outcomes[3].code, Some(0));
16557        assert_eq!(outcomes[3].output_tail, "success");
16558        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16559    }
16560
16561    /// The addendum's second gap: a `verify.gate` command running for real
16562    /// wall-clock time had nothing at all to show for it in `active` before
16563    /// `run_commands` learned to record it — a run could sit in `Gating` for
16564    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
16565    /// what was actually happening. Proven with a genuinely still-running
16566    /// command, not just a before/after check on the final state: a poller
16567    /// task reads the same `run.json` `gate()` is writing, the same way the
16568    /// phone or `magi show` would, while the shell command is still blocked
16569    /// on its own release marker.
16570    #[tokio::test]
16571    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16572        crate::run::pin_test_home();
16573
16574        let tmp = tempfile::tempdir().expect("tempdir");
16575        let repo = tmp.path().join("repo");
16576        std::fs::create_dir_all(&repo).unwrap();
16577        init_repo(&repo);
16578
16579        let mut config = Config::default();
16580        config.verify.gate = vec![
16581            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16582             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16583                .to_owned(),
16584        ];
16585
16586        let mut state = RunState::new(
16587            repo.clone(),
16588            "main".to_owned(),
16589            "deadbeef".to_owned(),
16590            "task".to_owned(),
16591            config,
16592        );
16593        let run_id = state.id.clone();
16594        state.candidates = vec![Candidate {
16595            index: 0,
16596            label: 'A',
16597            agent: "alpha".to_owned(),
16598            branch: "does-not-exist".to_owned(),
16599            worktree: repo.clone(),
16600            summary: String::new(),
16601            stat: String::new(),
16602            files: 0,
16603            commits: 0,
16604            empty: false,
16605            failed: None,
16606            verified_noop: None,
16607            duration_ms: 0,
16608            folded: false,
16609        }];
16610        state.tally = Some(Tally {
16611            first_choice: BTreeMap::from([('A', 1)]),
16612            borda: BTreeMap::new(),
16613            winner: 'A',
16614            rankings: 1,
16615            unanimous_initial: true,
16616            deliberated: false,
16617            changed_votes: 0,
16618            unanimous_final: true,
16619            tie_break: None,
16620            judges: 0,
16621            present: 0,
16622            quorum: 0,
16623            met_quorum: true,
16624            uncontested: Some("only candidate A produced a change".to_owned()),
16625        });
16626        state.reviews = vec![ReviewRound {
16627            round: 1,
16628            head: "deadbeef".to_owned(),
16629            verified_head: None,
16630            verified_at: None,
16631            reviews: Vec::new(),
16632            e2e: Vec::new(),
16633            fix: None,
16634            blocking: 0,
16635            answered: 0,
16636            expected: 0,
16637            clean: true,
16638            verify_retried: false,
16639            e2e_deferred: false,
16640            e2e_defer_reason: None,
16641            progressed: false,
16642            vote_split: false,
16643            reconsideration: Vec::new(),
16644            verdict: None,
16645        }];
16646
16647        let mut runner = Runner {
16648            state,
16649            roles: ResolvedRoles {
16650                implementers: Vec::new(),
16651                judges: Vec::new(),
16652                reviewers: Vec::new(),
16653                fixer: None,
16654                conductor: conductor(),
16655                implementer_roster: Vec::new(),
16656                judge_roster: Vec::new(),
16657                reviewer_roster: Vec::new(),
16658            },
16659            sem: Arc::new(Semaphore::new(1)),
16660            pause: Pause::new(),
16661            interrupt: Pause::new(),
16662        };
16663
16664        let started_marker = repo.join("started.marker");
16665        let release_marker = repo.join("release.marker");
16666        let poller = tokio::spawn(async move {
16667            // Bounded so a regression that never records the task entry
16668            // fails this test in seconds instead of hanging the suite —
16669            // the same shape `a_park_requested_while_a_seat_is_mid_call_
16670            // does_not_cut_it_short` uses for the same reason.
16671            for _ in 0..100 {
16672                if started_marker.exists()
16673                    && let Ok(s) = crate::run::RunState::load(&run_id)
16674                    && let Some(a) = s.active.get("gate")
16675                {
16676                    std::fs::write(&release_marker, b"go").expect("release marker");
16677                    return Some(a.clone());
16678                }
16679                tokio::time::sleep(Duration::from_millis(50)).await;
16680            }
16681            None
16682        });
16683
16684        runner.gate().await.expect("gate");
16685        let captured = poller.await.expect("poller task");
16686        let captured = captured.expect(
16687            "the poller never saw a `gate` task entry in run.json while the command was \
16688             still blocked on its own release marker",
16689        );
16690
16691        assert_eq!(captured.task.as_deref(), Some("gate"));
16692        assert_eq!(captured.node, "gate");
16693        assert_eq!(captured.index, Some(1));
16694        assert_eq!(captured.total, Some(1));
16695        assert!(
16696            captured
16697                .command
16698                .as_deref()
16699                .is_some_and(|c| c.contains("started.marker")),
16700            "{captured:?}"
16701        );
16702
16703        assert!(
16704            runner.state.active.is_empty(),
16705            "the entry must be cleared once the command actually finished: {:?}",
16706            runner.state.active
16707        );
16708        assert!(runner.state.gate_ran);
16709        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16710    }
16711
16712    /// The hand-off over a blocking finding a reviewer rejected on leaves a
16713    /// record for `land`; one with only a Minor, or no reject, leaves none.
16714    #[tokio::test]
16715    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16716        use crate::verdict::{Finding, ReviewVote, Severity};
16717        crate::run::pin_test_home();
16718        let tmp = tempfile::tempdir().expect("tempdir");
16719        let repo = tmp.path().join("repo");
16720        std::fs::create_dir_all(&repo).unwrap();
16721        init_repo(&repo);
16722
16723        for (severity, vote, expect) in [
16724            (Severity::Major, ReviewVote::Reject, true),
16725            (Severity::Minor, ReviewVote::Reject, false),
16726            (Severity::Major, ReviewVote::Approve, false),
16727        ] {
16728            let mut round = review_round(false, 1, 1, 1, false, true);
16729            round.reviews = vec![ReviewRecord {
16730                reviewer: 1,
16731                agent: "alpha".to_owned(),
16732                summary: String::new(),
16733                findings: vec![Finding {
16734                    id: "R1-1-1".to_owned(),
16735                    severity,
16736                    file: None,
16737                    line: None,
16738                    title: "t".to_owned(),
16739                    detail: String::new(),
16740                }],
16741                vote: Some(vote),
16742                failed: None,
16743                duration_ms: 0,
16744                attempts: 0,
16745            }];
16746            let mut state = RunState::new(
16747                repo.clone(),
16748                "main".to_owned(),
16749                "deadbeef".to_owned(),
16750                "task".to_owned(),
16751                Config::default(),
16752            );
16753            state.reviews = vec![round];
16754            let mut runner = Runner {
16755                state,
16756                roles: ResolvedRoles {
16757                    implementers: Vec::new(),
16758                    judges: Vec::new(),
16759                    reviewers: Vec::new(),
16760                    fixer: None,
16761                    conductor: conductor(),
16762                    implementer_roster: Vec::new(),
16763                    judge_roster: Vec::new(),
16764                    reviewer_roster: Vec::new(),
16765                },
16766                sem: Arc::new(Semaphore::new(1)),
16767                pause: Pause::new(),
16768                interrupt: Pause::new(),
16769            };
16770            let shell = runner.state.config.shell();
16771            runner
16772                .stop_reviewing("round budget spent", &shell, &repo)
16773                .await
16774                .expect("stop_reviewing");
16775            assert_eq!(runner.state.status, RunStatus::Gating);
16776            assert_eq!(
16777                runner.state.contested_handoff.is_some(),
16778                expect,
16779                "{severity:?} + {vote:?}"
16780            );
16781        }
16782    }
16783
16784    /// The shape the incident this whole fix responds to actually had: the
16785    /// round budget spent, the last round's own e2e blocked on the shared
16786    /// build cache (held here by a live pid — this test process — exactly
16787    /// `cache`'s own unit tests' pattern for "another owner, still alive"
16788    /// without forking a process). `stop_reviewing` must retry it — not
16789    /// silently leave the round looking untouched (the catch-up-only half of
16790    /// the bug), and not read the contention as a red `e2e` and block the
16791    /// run on it (the other half). Called directly, the same way
16792    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
16793    /// above exercises `gate`, so this never needs a real cargo build to
16794    /// reach: the lease is never released, so `with_cache_lease` never gets
16795    /// past acquiring it into anything that would need a real workspace.
16796    #[tokio::test]
16797    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16798        crate::run::pin_test_home();
16799        let home = crate::run::home();
16800
16801        let tmp = tempfile::tempdir().expect("tempdir");
16802        let repo = tmp.path().join("repo");
16803        std::fs::create_dir_all(&repo).unwrap();
16804        init_repo(&repo);
16805        let head = crate::git::rev_parse(&repo, "HEAD")
16806            .await
16807            .expect("rev-parse");
16808        // Unique to this test, so holding its lease cannot collide with
16809        // another test sharing the same process-wide `home`.
16810        let cache_dir = tmp.path().join("target");
16811
16812        let mut config = Config::default();
16813        config.verify.e2e = vec![format!(
16814            "CARGO_TARGET_DIR='{}' test -f README.md",
16815            cache_dir.display()
16816        )];
16817        config.graph.review_rounds = 1;
16818        // Bounded so a regression that does start waiting fails the test in
16819        // seconds, not hangs it.
16820        config.graph.timeout_verify = Some(2);
16821        config.graph.timeout_resource_wait = 2;
16822
16823        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16824        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16825            .expect("no io error acquiring directly")
16826        {
16827            crate::cache::AcquireOutcome::Acquired(g) => g,
16828            crate::cache::AcquireOutcome::Busy(b) => {
16829                panic!("expected the direct acquire to win the lease first: {b:?}")
16830            }
16831        };
16832
16833        let mut state = RunState::new(
16834            repo.clone(),
16835            "main".to_owned(),
16836            head.clone(),
16837            "task".to_owned(),
16838            config,
16839        );
16840        state.candidates = vec![Candidate {
16841            index: 0,
16842            label: 'A',
16843            agent: "alpha".to_owned(),
16844            branch: "does-not-exist".to_owned(),
16845            worktree: repo.clone(),
16846            summary: String::new(),
16847            stat: String::new(),
16848            files: 0,
16849            commits: 0,
16850            empty: false,
16851            failed: None,
16852            verified_noop: None,
16853            duration_ms: 0,
16854            folded: false,
16855        }];
16856        state.tally = Some(Tally {
16857            first_choice: BTreeMap::from([('A', 1)]),
16858            borda: BTreeMap::new(),
16859            winner: 'A',
16860            rankings: 1,
16861            unanimous_initial: true,
16862            deliberated: false,
16863            changed_votes: 0,
16864            unanimous_final: true,
16865            tie_break: None,
16866            judges: 0,
16867            present: 0,
16868            quorum: 0,
16869            met_quorum: true,
16870            uncontested: Some("only candidate A produced a change".to_owned()),
16871        });
16872        // The round budget's last round, deferred: `needs_catchup_run`'s
16873        // other trigger. `stop_reviewing`'s retry machinery must treat this
16874        // exactly like a resource-blocked attempt once it actually runs.
16875        state.reviews = vec![ReviewRound {
16876            round: 1,
16877            head: head.clone(),
16878            verified_head: None,
16879            verified_at: None,
16880            reviews: Vec::new(),
16881            e2e: Vec::new(),
16882            fix: None,
16883            blocking: 1,
16884            answered: 1,
16885            expected: 1,
16886            clean: false,
16887            verify_retried: false,
16888            e2e_deferred: true,
16889            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16890            progressed: false,
16891            vote_split: false,
16892            reconsideration: Vec::new(),
16893            verdict: None,
16894        }];
16895
16896        let mut runner = Runner {
16897            state,
16898            roles: ResolvedRoles {
16899                implementers: Vec::new(),
16900                judges: Vec::new(),
16901                reviewers: Vec::new(),
16902                fixer: None,
16903                conductor: conductor(),
16904                implementer_roster: Vec::new(),
16905                judge_roster: Vec::new(),
16906                reviewer_roster: Vec::new(),
16907            },
16908            sem: Arc::new(Semaphore::new(1)),
16909            pause: Pause::new(),
16910            interrupt: Pause::new(),
16911        };
16912
16913        let shell = runner.state.config.shell();
16914        runner
16915            .stop_reviewing("round budget spent", &shell, &repo)
16916            .await
16917            .expect("stop_reviewing");
16918
16919        let last = runner.state.reviews.last().expect("round record");
16920        assert_eq!(
16921            last.e2e_status(),
16922            E2eStatus::ResourceBlocked,
16923            "the shared cache is still held; the attempt must read as blocked, not deferred or \
16924             failed: {last:?}"
16925        );
16926        assert_eq!(
16927            last.verified_head.as_deref(),
16928            Some(head.as_str()),
16929            "which commit this attempt targeted is known even though nothing finished checking \
16930             it"
16931        );
16932        let first_attempt_at = last
16933            .verified_at
16934            .expect("when this attempt ran is known too");
16935        assert_ne!(
16936            runner.state.status,
16937            RunStatus::Blocked,
16938            "contention is evidence about the machine, not the patch — it must not settle the \
16939             run as blocked: {:?}",
16940            runner.state.status
16941        );
16942        assert!(
16943            !runner
16944                .state
16945                .events
16946                .iter()
16947                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16948            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16949            runner.state.events
16950        );
16951
16952        // The cache is still held: a later reentry must retry the same
16953        // round's verification again — not leave it looking exactly as
16954        // untouched as the first blocked attempt, which is indistinguishable
16955        // from never having tried again at all.
16956        runner
16957            .stop_reviewing("round budget spent", &shell, &repo)
16958            .await
16959            .expect("stop_reviewing retry");
16960        assert_eq!(
16961            runner.state.reviews.len(),
16962            1,
16963            "no new round was started: {:?}",
16964            runner.state.reviews
16965        );
16966        let last = runner.state.reviews.last().expect("round record");
16967        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16968        assert!(
16969            last.verified_at.expect("still known") > first_attempt_at,
16970            "a second reentry must be a fresh attempt, not a stale copy of the first"
16971        );
16972        assert_ne!(runner.state.status, RunStatus::Blocked);
16973
16974        held.release();
16975    }
16976
16977    #[cfg(unix)]
16978    #[tokio::test]
16979    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
16980        ask_test_home();
16981        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
16982            let tmp = tempfile::tempdir().unwrap();
16983            let repo = tmp.path().join("repo");
16984            std::fs::create_dir(&repo).unwrap();
16985            init_repo(&repo);
16986            let run_git = |args: &[&str]| {
16987                let out = std::process::Command::new("git")
16988                    .args(args)
16989                    .current_dir(&repo)
16990                    .output()
16991                    .unwrap();
16992                assert!(
16993                    out.status.success(),
16994                    "{args:?}: {}",
16995                    String::from_utf8_lossy(&out.stderr)
16996                );
16997            };
16998            let origin = tmp.path().join("origin.git");
16999            run_git(&["init", "--bare", origin.to_str().unwrap()]);
17000            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17001            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17002            run_git(&["add", "spec.rs"]);
17003            run_git(&["commit", "-m", "initial spec"]);
17004            run_git(&["push", "origin", "main"]);
17005            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17006            run_git(&["checkout", "-b", "fix/review"]);
17007            std::fs::write(
17008                repo.join("check.rs"),
17009                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17010            )
17011            .unwrap();
17012            run_git(&["add", "check.rs"]);
17013            run_git(&["commit", "-m", "use spec"]);
17014            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17015            run_git(&["checkout", "main"]);
17016            std::fs::write(
17017                repo.join("spec.rs"),
17018                "struct AgentSpec { max_turn_secs: u64 }\n",
17019            )
17020            .unwrap();
17021            run_git(&["add", "spec.rs"]);
17022            run_git(&["commit", "-m", "extend spec"]);
17023            run_git(&["push", "origin", "main"]);
17024            run_git(&["checkout", "fix/review"]);
17025
17026            let mut runner = runner_at(RunStatus::Reviewing);
17027            runner.state.repo = repo.clone();
17028            runner.state.base_commit = base;
17029            runner.state.pr_head = Some("fix/review".into());
17030            runner.state.config.graph.review_rounds = max_rounds;
17031            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17032            runner.state.config.graph.sessions = false;
17033            runner.state.config.disk.min_free_bytes = 0;
17034            runner.state.config.verify.e2e =
17035                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17036            runner.state.candidates = vec![Candidate {
17037                index: 0,
17038                label: 'A',
17039                agent: "fixture".into(),
17040                branch: "fix/review".into(),
17041                worktree: repo.clone(),
17042                summary: String::new(),
17043                stat: String::new(),
17044                files: 1,
17045                commits: 1,
17046                empty: false,
17047                failed: None,
17048                verified_noop: None,
17049                duration_ms: 0,
17050                folded: false,
17051            }];
17052            runner.state.tally = Some(Tally {
17053                first_choice: BTreeMap::new(),
17054                borda: BTreeMap::new(),
17055                winner: 'A',
17056                rankings: 0,
17057                unanimous_initial: true,
17058                deliberated: false,
17059                changed_votes: 0,
17060                unanimous_final: true,
17061                tie_break: None,
17062                judges: 0,
17063                present: 0,
17064                quorum: 0,
17065                met_quorum: true,
17066                uncontested: Some("review-only".into()),
17067            });
17068            let mut last = review_round(false, 0, 1, 1, false, false);
17069            last.head = reviewed.clone();
17070            last.e2e_deferred = true;
17071            last.e2e[0].resource_blocked = true;
17072            last.e2e[0].output_tail = "waiting for the shared build cache".into();
17073            if max_rounds == 1 {
17074                last.e2e.clear();
17075            }
17076            runner.state.reviews = vec![last];
17077            let calls = tmp.path().join("fix-calls");
17078            let mut fixer = spec("repair");
17079            fixer
17080                .env
17081                .insert("FIX_CALLS".into(), calls.display().to_string());
17082            fixer.command = vec!["sh".into(), "-c".into(), r#"
17083                echo fix >> "$FIX_CALLS"
17084                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17085                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17086                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17087                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17088            "#.into()];
17089            let mut reviewer = spec("reviewer");
17090            reviewer.command = vec![
17091                "sh".into(),
17092                "-c".into(),
17093                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17094                    .into(),
17095            ];
17096            runner.roles.fixer = Some(vec![fixer]);
17097            runner.roles.reviewers = vec![reviewer];
17098
17099            runner.sync_to_base().await.unwrap();
17100            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17101            assert_ne!(rebased, reviewed);
17102            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17103            if reentry {
17104                // Simulate a crash after catch-up evidence was saved but before
17105                // dispatching the fixer. No dedicated pending-repair field exists.
17106                let shell = runner.state.config.shell();
17107                runner
17108                    .stop_unresolved_review("deferred verification", &shell, &repo)
17109                    .await
17110                    .unwrap();
17111                assert!(runner.catchup_repairable());
17112                assert!(!calls.exists());
17113                runner.state = RunState::load(&runner.state.id).unwrap();
17114                assert!(runner.catchup_repairable());
17115            }
17116            runner.review_loop().await.unwrap();
17117            let first = &runner.state.reviews[0];
17118            assert_eq!(first.head, reviewed);
17119            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17120            assert!(
17121                first.verify_retried,
17122                "a reproduced build failure gets exactly one retry: {:?}",
17123                first.e2e
17124            );
17125            assert!(first.e2e[0].output_tail.contains("E0063"));
17126            assert!(!runner.catchup_repairable());
17127            if max_rounds == 2 {
17128                assert_eq!(
17129                    runner.state.status,
17130                    RunStatus::Gating,
17131                    "{}",
17132                    crate::report::run(&runner.state)
17133                );
17134                assert_eq!(runner.state.reviews.len(), 2);
17135                assert!(first.fix.as_ref().unwrap().committed);
17136                assert!(runner.state.reviews[1].clean);
17137                assert_eq!(
17138                    runner
17139                        .state
17140                        .reviews
17141                        .iter()
17142                        .filter(|r| r.fix.is_some())
17143                        .count(),
17144                    1
17145                );
17146                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17147                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17148            } else {
17149                assert_eq!(runner.state.status, RunStatus::Blocked);
17150                assert!(first.fix.is_none());
17151                assert!(!calls.exists());
17152                let recovery = crate::recovery::assess(
17153                    "prior",
17154                    Some(&runner.state),
17155                    &runner.landing_base(),
17156                    crate::recovery::Relation::Diverged,
17157                );
17158                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17159                let reasons = recovery.reasons.join("\n");
17160                assert!(
17161                    reasons.contains(&reviewed) && reasons.contains(&rebased),
17162                    "{reasons}"
17163                );
17164                assert!(
17165                    reasons.contains("base sync to")
17166                        && reasons.contains("additional review is required"),
17167                    "{reasons}"
17168                );
17169            }
17170        }
17171    }
17172
17173    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
17174    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
17175    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
17176    /// own top-of-function fast path (`review_conclusion`) correctly reads
17177    /// this shape as `None` rather than guessing `Blocked`, and the loop's
17178    /// own `for` range is empty once the round budget is spent, so
17179    /// `review_loop` must retry the check itself rather than silently doing
17180    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
17181    /// above exercises directly, but through `review_loop`'s own entry point
17182    /// this time, proving the wiring between the two rather than just the
17183    /// retry logic in isolation.
17184    #[tokio::test]
17185    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17186        crate::run::pin_test_home();
17187        let home = crate::run::home();
17188
17189        let tmp = tempfile::tempdir().expect("tempdir");
17190        let repo = tmp.path().join("repo");
17191        std::fs::create_dir_all(&repo).unwrap();
17192        init_repo(&repo);
17193        let head = crate::git::rev_parse(&repo, "HEAD")
17194            .await
17195            .expect("rev-parse");
17196        let cache_dir = tmp.path().join("target");
17197
17198        let mut config = Config::default();
17199        config.verify.e2e = vec![format!(
17200            "CARGO_TARGET_DIR='{}' test -f README.md",
17201            cache_dir.display()
17202        )];
17203        config.graph.review_rounds = 1;
17204        config.graph.timeout_verify = Some(2);
17205        config.graph.timeout_resource_wait = 2;
17206
17207        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17208        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17209            .expect("no io error acquiring directly")
17210        {
17211            crate::cache::AcquireOutcome::Acquired(g) => g,
17212            crate::cache::AcquireOutcome::Busy(b) => {
17213                panic!("expected the direct acquire to win the lease first: {b:?}")
17214            }
17215        };
17216
17217        let mut state = RunState::new(
17218            repo.clone(),
17219            "main".to_owned(),
17220            head.clone(),
17221            "task".to_owned(),
17222            config,
17223        );
17224        state.candidates = vec![Candidate {
17225            index: 0,
17226            label: 'A',
17227            agent: "alpha".to_owned(),
17228            branch: "does-not-exist".to_owned(),
17229            worktree: repo.clone(),
17230            summary: String::new(),
17231            stat: String::new(),
17232            files: 0,
17233            commits: 0,
17234            empty: false,
17235            failed: None,
17236            verified_noop: None,
17237            duration_ms: 0,
17238            folded: false,
17239        }];
17240        state.tally = Some(Tally {
17241            first_choice: BTreeMap::from([('A', 1)]),
17242            borda: BTreeMap::new(),
17243            winner: 'A',
17244            rankings: 1,
17245            unanimous_initial: true,
17246            deliberated: false,
17247            changed_votes: 0,
17248            unanimous_final: true,
17249            tie_break: None,
17250            judges: 0,
17251            present: 0,
17252            quorum: 0,
17253            met_quorum: true,
17254            uncontested: Some("only candidate A produced a change".to_owned()),
17255        });
17256        // The exact shape a prior process's `stop_reviewing` would have left
17257        // on disk: the round budget's last round, a real attempt already
17258        // made and already resource-blocked.
17259        state.reviews = vec![ReviewRound {
17260            round: 1,
17261            head: head.clone(),
17262            verified_head: Some(head.clone()),
17263            verified_at: Some(jiff::Timestamp::now()),
17264            reviews: Vec::new(),
17265            e2e: vec![CommandOutcome {
17266                command: format!(
17267                    "CARGO_TARGET_DIR='{}' test -f README.md",
17268                    cache_dir.display()
17269                ),
17270                code: None,
17271                output_tail: "waiting for the shared build cache".to_owned(),
17272                duration_ms: 0,
17273                resource_blocked: true,
17274            }],
17275            fix: None,
17276            blocking: 1,
17277            answered: 1,
17278            expected: 1,
17279            clean: false,
17280            verify_retried: false,
17281            e2e_deferred: false,
17282            e2e_defer_reason: None,
17283            progressed: false,
17284            vote_split: false,
17285            reconsideration: Vec::new(),
17286            verdict: None,
17287        }];
17288
17289        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17290        let mut runner = Runner {
17291            state,
17292            roles: ResolvedRoles {
17293                implementers: Vec::new(),
17294                judges: Vec::new(),
17295                reviewers: Vec::new(),
17296                fixer: None,
17297                conductor: conductor(),
17298                implementer_roster: Vec::new(),
17299                judge_roster: Vec::new(),
17300                reviewer_roster: Vec::new(),
17301            },
17302            sem: Arc::new(Semaphore::new(1)),
17303            pause: Pause::new(),
17304            interrupt: Pause::new(),
17305        };
17306
17307        // The lease is still held throughout, so this reentry's own retry is
17308        // also contended — proving `review_loop` actually tried again (not
17309        // that it happened to succeed) is what the timestamp comparison
17310        // below is for.
17311        runner.review_loop().await.expect("review_loop");
17312
17313        assert_eq!(
17314            runner.state.reviews.len(),
17315            1,
17316            "no new round was started on top of the unresolved one: {:?}",
17317            runner.state.reviews
17318        );
17319        let last = &runner.state.reviews[0];
17320        assert_eq!(
17321            last.e2e_status(),
17322            E2eStatus::ResourceBlocked,
17323            "still contended: {last:?}"
17324        );
17325        assert!(
17326            last.verified_at.expect("still known") > first_attempt_at,
17327            "review_loop must have actually retried the check, not left it exactly as found"
17328        );
17329        assert_ne!(
17330            runner.state.status,
17331            RunStatus::Blocked,
17332            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17333            runner.state.status
17334        );
17335
17336        held.release();
17337    }
17338
17339    #[tokio::test]
17340    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17341        crate::run::pin_test_home();
17342        for status in [RunStatus::Blocked, RunStatus::Gating] {
17343            let mut runner = runner_at(status);
17344            runner.state.config.graph.land = true;
17345            runner.state.gate_ran = true;
17346            runner.state.merge = Some(MergeOutcome {
17347                mode: MergeMode::Pr,
17348                ok: false,
17349                detail: "the fixer produced no commit".into(),
17350                empty: false,
17351            });
17352            let url = "https://example.invalid/o/r/pull/1";
17353            runner.state.pr = Some(crate::run::PrRecord {
17354                url: url.into(),
17355                number: 1,
17356                state: "open".into(),
17357                checks: "red".into(),
17358                round: 1,
17359                rounds: 1,
17360                red_at_merge: Vec::new(),
17361            });
17362            runner.state.save().unwrap();
17363            // The nonexistent repository makes observation fail before a forge
17364            // command can run. We still must reach land, rather than silently
17365            // return from the graph with its old gate and merge records.
17366            runner.execute_graph().await.unwrap();
17367            assert_eq!(runner.state.status, RunStatus::Blocked);
17368            assert!(runner.state.land_sync_pending);
17369            assert!(runner.state.events.iter().any(|event|
17370                event.node == "land" && event.message == format!("watching {url}")));
17371            assert_eq!(
17372                runner.state.merge.as_ref().unwrap().detail,
17373                "the fixer produced no commit"
17374            );
17375        }
17376    }
17377
17378    #[tokio::test]
17379    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17380        crate::run::pin_test_home();
17381        let tmp = tempfile::tempdir().expect("tempdir");
17382        let repo = tmp.path().join("repo");
17383        std::fs::create_dir_all(&repo).unwrap();
17384        init_repo(&repo);
17385
17386        let mut config = Config::default();
17387        config.merge.mode = MergeMode::Pr;
17388        config.graph.land = true;
17389        config.graph.land_approval = false;
17390
17391        let mut state = RunState::new(
17392            repo.clone(),
17393            "main".to_owned(),
17394            "deadbeef".to_owned(),
17395            "task".to_owned(),
17396            config,
17397        );
17398        state.candidates = vec![Candidate {
17399            index: 0,
17400            label: 'A',
17401            agent: "alpha".to_owned(),
17402            branch: "does-not-exist".to_owned(),
17403            worktree: repo.clone(),
17404            summary: String::new(),
17405            stat: String::new(),
17406            files: 0,
17407            commits: 0,
17408            empty: false,
17409            failed: None,
17410            verified_noop: None,
17411            duration_ms: 0,
17412            folded: false,
17413        }];
17414        state.tally = Some(Tally {
17415            first_choice: BTreeMap::from([('A', 1)]),
17416            borda: BTreeMap::new(),
17417            winner: 'A',
17418            rankings: 1,
17419            unanimous_initial: true,
17420            deliberated: false,
17421            changed_votes: 0,
17422            unanimous_final: true,
17423            tie_break: None,
17424            judges: 0,
17425            present: 0,
17426            quorum: 0,
17427            met_quorum: true,
17428            uncontested: Some("only candidate A produced a change".to_owned()),
17429        });
17430        state.reviews = vec![ReviewRound {
17431            round: 1,
17432            head: "deadbeef".to_owned(),
17433            verified_head: None,
17434            verified_at: None,
17435            reviews: Vec::new(),
17436            e2e: Vec::new(),
17437            fix: None,
17438            blocking: 0,
17439            answered: 0,
17440            expected: 0,
17441            clean: true,
17442            verify_retried: false,
17443            e2e_deferred: false,
17444            e2e_defer_reason: None,
17445            progressed: false,
17446            vote_split: false,
17447            reconsideration: Vec::new(),
17448            verdict: None,
17449        }];
17450        state.gate = vec![CommandOutcome {
17451            command: "test".to_owned(),
17452            code: Some(0),
17453            output_tail: String::new(),
17454            duration_ms: 0,
17455            resource_blocked: false,
17456        }];
17457        state.gate_ran = true;
17458        // A first pass through `merge` already pushed and opened this pull
17459        // request; `status` is `Landing` because a previous call into `land`
17460        // parked or was interrupted before it reached a terminal outcome.
17461        state.status = RunStatus::Landing;
17462        state.merge = Some(MergeOutcome {
17463            mode: MergeMode::Pr,
17464            ok: true,
17465            detail: "https://example.invalid/x/y/pull/1".to_owned(),
17466            empty: false,
17467        });
17468
17469        // The Landing-resume shortcut calls `run_land` directly rather than
17470        // through `merge`, which is exactly the call site that used to skip
17471        // `settle_questions` - see the fixture below.
17472        ask_test_home();
17473        // A real resume loads a persisted run; driver ownership is checked
17474        // against that record before the Landing shortcut is entered.
17475        state.save().expect("save landing run");
17476        let store = ask::Questions::open();
17477        let q = ask_open_question(&store, &state.id);
17478
17479        let mut runner = Runner {
17480            state,
17481            roles: ResolvedRoles {
17482                implementers: Vec::new(),
17483                judges: Vec::new(),
17484                reviewers: Vec::new(),
17485                fixer: None,
17486                conductor: conductor(),
17487                implementer_roster: Vec::new(),
17488                judge_roster: Vec::new(),
17489                reviewer_roster: Vec::new(),
17490            },
17491            sem: Arc::new(Semaphore::new(1)),
17492            pause: Pause::new(),
17493            interrupt: Pause::new(),
17494        };
17495
17496        // `execute`, not `merge` directly: the Landing-resume shortcut lives
17497        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
17498        // exactly because `review_loop` would otherwise clobber the marker
17499        // first.
17500        runner.execute().await.expect("execute");
17501
17502        assert_eq!(
17503            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17504            Some("https://example.invalid/x/y/pull/1"),
17505            "reentry must not push again or open a second pull request over the \
17506             one `land` is already watching"
17507        );
17508        assert_ne!(
17509            runner.state.status,
17510            RunStatus::Landing,
17511            "land could not actually reach the fake pull request, so it must \
17512             have given up rather than left the run silently parked forever"
17513        );
17514        // `land` could not reach the fake pull request, so it gave up into
17515        // `Blocked` - still resumable, so the question must not have been
17516        // swept just because this branch now also calls `settle_questions`.
17517        assert_eq!(runner.state.status, RunStatus::Blocked);
17518        assert!(
17519            store.get(&q.id).unwrap().status.open(),
17520            "Blocked is still alive; settle_questions must have been a no-op here"
17521        );
17522    }
17523
17524    fn state_with_round(round: ReviewRound) -> RunState {
17525        let mut s = RunState::new(
17526            PathBuf::from("/repo"),
17527            "main".to_owned(),
17528            "abc1234".to_owned(),
17529            "add retries".to_owned(),
17530            Config::default(),
17531        );
17532        s.reviews = vec![round];
17533        s
17534    }
17535
17536    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17537        crate::verdict::Finding {
17538            id: id.to_owned(),
17539            severity,
17540            file: None,
17541            line: None,
17542            title: title.to_owned(),
17543            detail: String::new(),
17544        }
17545    }
17546
17547    #[test]
17548    fn pr_body_names_open_findings_and_declined_ones() {
17549        let round = ReviewRound {
17550            round: 2,
17551            head: "deadbee".to_owned(),
17552            verified_head: None,
17553            verified_at: None,
17554            reviews: vec![ReviewRecord {
17555                attempts: 0,
17556                reviewer: 1,
17557                agent: "alpha".to_owned(),
17558                summary: String::new(),
17559                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17560                vote: None,
17561                failed: None,
17562                duration_ms: 0,
17563            }],
17564            e2e: vec![CommandOutcome {
17565                command: "cargo test".to_owned(),
17566                code: Some(0),
17567                output_tail: String::new(),
17568                duration_ms: 0,
17569                resource_blocked: false,
17570            }],
17571            verify_retried: false,
17572            e2e_deferred: false,
17573            e2e_defer_reason: None,
17574            fix: Some(FixRecord {
17575                agent: "alpha".to_owned(),
17576                addressed: Vec::new(),
17577                rejected: vec![crate::verdict::Rejection {
17578                    id: "R1-1-1".to_owned(),
17579                    why: "not reachable from any caller".to_owned(),
17580                }],
17581                notes: String::new(),
17582                committed: true,
17583                failed: None,
17584                duration_ms: 0,
17585                continuation: None,
17586            }),
17587            blocking: 0,
17588            answered: 1,
17589            expected: 1,
17590            clean: false,
17591            progressed: true,
17592            vote_split: false,
17593            reconsideration: Vec::new(),
17594            verdict: None,
17595        };
17596        let state = state_with_round(round);
17597        let body = pr_message(&state, 'A').body;
17598
17599        assert!(body.contains("add retries"), "the task must still be there");
17600        assert!(body.contains("R2-1-1"), "{body}");
17601        assert!(!body.contains("unused import"), "{body}");
17602        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17603        assert!(
17604            !body.contains("not reachable from any caller"),
17605            "the reason it was declined: {body}"
17606        );
17607    }
17608
17609    #[test]
17610    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17611        let round = ReviewRound {
17612            round: 1,
17613            head: "deadbee".to_owned(),
17614            verified_head: None,
17615            verified_at: None,
17616            reviews: vec![ReviewRecord {
17617                attempts: 0,
17618                reviewer: 1,
17619                agent: "alpha".to_owned(),
17620                summary: String::new(),
17621                findings: Vec::new(),
17622                vote: None,
17623                failed: None,
17624                duration_ms: 0,
17625            }],
17626            e2e: Vec::new(),
17627            verify_retried: false,
17628            e2e_deferred: false,
17629            e2e_defer_reason: None,
17630            fix: None,
17631            blocking: 0,
17632            answered: 1,
17633            expected: 1,
17634            clean: true,
17635            progressed: false,
17636            vote_split: false,
17637            reconsideration: Vec::new(),
17638            verdict: None,
17639        };
17640        let state = state_with_round(round);
17641        let body = pr_message(&state, 'A').body;
17642        assert!(!body.contains("Open review findings"), "{body}");
17643        assert!(!body.contains("Declined"), "{body}");
17644    }
17645
17646    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17647        let mut state = RunState::new(
17648            PathBuf::from("/repo"),
17649            "main".to_owned(),
17650            "abc1234".to_owned(),
17651            instruction.to_owned(),
17652            Config::default(),
17653        );
17654        state.candidates.push(Candidate {
17655            index: 0,
17656            label: 'A',
17657            agent: "alpha".to_owned(),
17658            branch: "magi/x/A".to_owned(),
17659            worktree: PathBuf::from("/wt"),
17660            summary: summary.to_owned(),
17661            stat: String::new(),
17662            files: 1,
17663            commits: 1,
17664            empty: false,
17665            failed: None,
17666            verified_noop: None,
17667            folded: false,
17668            duration_ms: 0,
17669        });
17670        state
17671    }
17672
17673    fn review_state(subjects: &[&str]) -> RunState {
17674        let mut state = state_with_summary(
17675            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17676            "",
17677        );
17678        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17679        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17680        state
17681    }
17682
17683    /// A branch rebased onto a main that moved past the recorded
17684    /// `base_commit` is titled from its own first commit, never main's.
17685    #[tokio::test]
17686    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17687        ask_test_home();
17688        let tmp = tempfile::tempdir().unwrap();
17689        let repo = tmp.path().join("repo");
17690        std::fs::create_dir_all(&repo).unwrap();
17691        init_repo(&repo);
17692        let origin = tmp.path().join("origin.git");
17693        let g = |dir: &Path, args: &[&str]| {
17694            let out = std::process::Command::new("git")
17695                .args(args)
17696                .current_dir(dir)
17697                .quiet()
17698                .output()
17699                .expect("spawn git");
17700            assert!(
17701                out.status.success(),
17702                "git {args:?}: {}",
17703                String::from_utf8_lossy(&out.stderr)
17704            );
17705        };
17706        g(
17707            tmp.path(),
17708            &[
17709                "clone",
17710                "--bare",
17711                "-q",
17712                repo.to_str().unwrap(),
17713                origin.to_str().unwrap(),
17714            ],
17715        );
17716        g(
17717            &repo,
17718            &["remote", "add", "origin", origin.to_str().unwrap()],
17719        );
17720        let c1 = git::rev_parse(&repo, "main").await.unwrap();
17721
17722        // Main moves on; the branch is built on top of the new main.
17723        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17724        g(&repo, &["add", "-A"]);
17725        g(
17726            &repo,
17727            &["commit", "-q", "-m", "chore(deps): update a crate"],
17728        );
17729        g(&repo, &["push", "-q", "origin", "main"]);
17730        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17731        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17732        g(&repo, &["add", "-A"]);
17733        g(
17734            &repo,
17735            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17736        );
17737        g(&repo, &["checkout", "-q", "main"]);
17738
17739        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17740        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17741        // Without a readable tracking ref the recorded base's merge base is used.
17742        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17743        assert_eq!(fallback, c1);
17744
17745        let mut state = review_state(&[
17746            "chore(deps): update a crate",
17747            "fix(daemon): apply a chosen action",
17748        ]);
17749        state.repo = repo.clone();
17750        state.base_branch = "main".to_owned();
17751        state.base_commit = c1;
17752        refresh_reviewed_commits(&mut state, "feat/own").await;
17753        assert_eq!(
17754            state.reviewed_commits,
17755            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17756        );
17757        assert_eq!(
17758            review_title(&state).as_deref(),
17759            Some("fix(daemon): apply a chosen action")
17760        );
17761        assert_eq!(
17762            leaked_subjects(&state, "feat/own").await,
17763            Some(vec!["chore(deps): update a crate".to_owned()])
17764        );
17765        // A stale tracking ref is still used when the fetch fails, but the
17766        // leak list is withheld.
17767        state.config.merge.remote = "nowhere".to_owned();
17768        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17769    }
17770
17771    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
17772    /// with `origin/main` one commit behind it, and a summarizing agent that
17773    /// prints whatever `reply` holds.
17774    struct SummaryFixture {
17775        _tmp: tempfile::TempDir,
17776        repo: PathBuf,
17777        reply: PathBuf,
17778        runner: Runner,
17779    }
17780
17781    fn summary_fixture() -> SummaryFixture {
17782        ask_test_home();
17783        let tmp = tempfile::tempdir().unwrap();
17784        let repo = tmp.path().join("repo");
17785        std::fs::create_dir_all(&repo).unwrap();
17786        init_repo(&repo);
17787        let origin = tmp.path().join("origin.git");
17788        let g = |args: &[&str]| {
17789            let out = std::process::Command::new("git")
17790                .args(args)
17791                .current_dir(&repo)
17792                .quiet()
17793                .output()
17794                .expect("spawn git");
17795            assert!(
17796                out.status.success(),
17797                "git {args:?}: {}",
17798                String::from_utf8_lossy(&out.stderr)
17799            );
17800        };
17801        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17802        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17803        g(&["push", "-q", "origin", "main"]);
17804        g(&["checkout", "-q", "-b", "feat/own"]);
17805        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17806        g(&["add", "-A"]);
17807        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17808        let reply = tmp.path().join("reply.txt");
17809        std::fs::write(
17810            &reply,
17811            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17812        )
17813        .unwrap();
17814        let mut runner = runner_at(RunStatus::Gating);
17815        let mut state = review_state(&["feat(own): add the own file"]);
17816        state.repo = repo.clone();
17817        state.base_branch = "main".to_owned();
17818        state.candidates[0].branch = "feat/own".to_owned();
17819        state.candidates[0].worktree = repo.clone();
17820        let mut author = spec("alpha");
17821        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17822        author
17823            .env
17824            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17825        state.config.agents = vec![author];
17826        runner.state = state;
17827        SummaryFixture {
17828            _tmp: tmp,
17829            repo,
17830            reply,
17831            runner,
17832        }
17833    }
17834
17835    async fn posted(f: &mut SummaryFixture) -> PrMessage {
17836        let winner = f.runner.state.candidates[0].clone();
17837        f.runner
17838            .guarded_pr_message(&winner, None, true)
17839            .await
17840            .unwrap()
17841            .expect("an English description never parks")
17842    }
17843
17844    fn clean_round_with_minor_findings() -> ReviewRound {
17845        let mut round = passing_round(2, None);
17846        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17847        round.reviews = vec![ReviewRecord {
17848            attempts: 0,
17849            reviewer: 1,
17850            agent: "alpha".into(),
17851            summary: "内部レビュー".into(),
17852            findings: vec![
17853                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17854                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17855            ],
17856            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17857            failed: None,
17858            duration_ms: 0,
17859        }];
17860        round
17861    }
17862
17863    #[tokio::test]
17864    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17865        for review_only in [false, true] {
17866            let mut f = summary_fixture();
17867            if !review_only {
17868                f.runner.state.candidates[0].agent = "alpha".into();
17869            }
17870            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17871            assert!(f.runner.state.reviews[0].clean);
17872            assert!(f.runner.state.reviews[0].fix.is_none());
17873            assert!(f.runner.state.open_findings().is_empty());
17874            assert_eq!(
17875                crate::pr_outcome::required_finding_ids(&f.runner.state),
17876                ["R2-1-1", "R2-1-2"]
17877            );
17878            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17879            let evidence = crate::pr_outcome::summary_evidence(
17880                &f.runner.state,
17881                crate::pr_outcome::FinalHead::At(&head),
17882            );
17883            assert!(
17884                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17885            );
17886            // The ordinary reply omits both IDs: no cached description is adopted.
17887            let omitted = posted(&mut f).await;
17888            assert!(omitted.body.contains("No description"));
17889            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17890            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17891            assert!(f.runner.state.change_summary.is_none());
17892            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17893            let described = posted(&mut f).await;
17894            assert!(described.body.contains("improve documentation for callers"));
17895            assert!(described.body.contains("remaining edge case"));
17896            assert!(described.body.contains("## Open review findings"));
17897            assert!(!described.body.contains("未修正"));
17898        }
17899    }
17900
17901    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17902        let mut f = summary_fixture();
17903        if !review_only {
17904            f.runner.state.candidates[0].agent = "alpha".into();
17905        }
17906        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17907        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17908        fixed
17909            .fix
17910            .as_mut()
17911            .unwrap()
17912            .rejected
17913            .push(crate::verdict::Rejection {
17914                id: "R1-1-1".into(),
17915                why: "呼び出されないコードのため辞退".into(),
17916            });
17917        let mut clean = clean_round_with_minor_findings();
17918        clean.verified_head = Some(head.clone());
17919        f.runner.state.reviews = vec![fixed, clean];
17920        ok_gate(&mut f.runner.state, Some(&head));
17921        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17922        f.runner.state.config.agents.push(scripted("beta", reply));
17923        f.runner.state.config.roles.language_judge =
17924            Some(crate::config::AgentChoice::One("beta".into()));
17925        f
17926    }
17927
17928    #[tokio::test]
17929    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17930        for review_only in [false, true] {
17931            let mut f = rewrite_fixture_with_findings(review_only,
17932                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17933            let m = posted(&mut f).await;
17934            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17935            let expected =
17936                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17937            assert!(m.body.contains(&expected));
17938            assert!(m.body.contains("## Review rounds"));
17939            assert!(
17940                m.body
17941                    .contains("Gate (`verify.gate`): passed on the final commit")
17942            );
17943            assert!(
17944                m.body
17945                    .contains("Tests (`verify.e2e`): passed on the final commit")
17946            );
17947            let c = f.runner.state.change_summary.as_ref().unwrap();
17948            assert!(
17949                m.body
17950                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17951            );
17952            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17953            assert!(crate::pr_outcome::covers_required_findings(
17954                &f.runner.state,
17955                &m.body
17956            ));
17957            assert!(m.body.contains("documentation remains incomplete"));
17958            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17959            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17960            assert!(f.runner.state.github_text.is_none());
17961        }
17962    }
17963
17964    #[tokio::test]
17965    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17966        for review_only in [false, true] {
17967            for reply in [
17968                "TITLE: fix: public description\nA short English description.",
17969                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17970            ] {
17971                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17972                let winner = f.runner.state.candidates[0].clone();
17973                let posted = f
17974                    .runner
17975                    .guarded_pr_message(&winner, None, true)
17976                    .await
17977                    .unwrap();
17978                assert!(posted.is_none());
17979                assert!(f.runner.state.parked);
17980                let withheld = f.runner.state.github_text.as_ref().unwrap();
17981                assert!(
17982                    withheld.rewrite.is_none(),
17983                    "an incomplete rewrite must not be selectable"
17984                );
17985                let q = ask::Questions::open()
17986                    .get(withheld.question.as_ref().unwrap())
17987                    .unwrap();
17988                assert!(
17989                    !q.choices
17990                        .contains(&crate::github_text::USE_REWRITE.to_owned())
17991                );
17992                assert!(
17993                    q.detail.contains("R1-1-1")
17994                        && q.detail.contains("R2-1-1")
17995                        && q.detail.contains("R2-1-2")
17996                );
17997            }
17998        }
17999    }
18000
18001    #[tokio::test]
18002    async fn japanese_reports_stay_local_in_both_public_run_modes() {
18003        for review_only in [false, true] {
18004            let mut f = summary_fixture();
18005            if !review_only {
18006                f.runner.state.candidates[0].agent = "alpha".into();
18007            }
18008            f.runner.state.instruction = "内部タスクの説明".into();
18009            f.runner.state.candidates[0].summary = "内部の候補報告".into();
18010            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18011            f.runner.state.reviews = vec![
18012                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18013                round_with_notes(2, Some("二回目の内部修正報告です。")),
18014                passing_round(3, Some(&head)),
18015            ];
18016            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18017            ok_gate(&mut f.runner.state, Some(&head));
18018            let captured = f.repo.join("public-prompt.txt");
18019            let author = &mut f.runner.state.config.agents[0];
18020            author.command = vec![
18021                "sh".into(),
18022                "-c".into(),
18023                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18024            ];
18025            author.env.insert(
18026                "PROMPT_FILE".into(),
18027                captured.to_string_lossy().into_owned(),
18028            );
18029            let m = posted(&mut f).await;
18030            assert!(m.body.contains("Adds `own.txt`"));
18031            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18032            assert!(m.body.contains("R1-1-1"));
18033            assert!(
18034                m.body
18035                    .contains("Gate (`verify.gate`): passed on the final commit")
18036            );
18037            assert!(
18038                m.body
18039                    .contains("Tests (`verify.e2e`): passed on the final commit")
18040            );
18041            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18042            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18043            let prompt = std::fs::read_to_string(captured).unwrap();
18044            assert!(prompt.contains(&format!(
18045                "git diff {} {head}",
18046                f.runner.state.change_summary.as_ref().unwrap().base
18047            )));
18048            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18049            assert_eq!(
18050                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18051                "二回目の内部修正報告です。"
18052            );
18053        }
18054    }
18055
18056    #[tokio::test]
18057    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18058        for review_only in [false, true] {
18059            let mut f = summary_fixture();
18060            if !review_only {
18061                f.runner.state.candidates[0].agent = "alpha".into();
18062            }
18063            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18064            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18065            std::fs::write(&f.reply, "no usable description").unwrap();
18066            let failed = posted(&mut f).await;
18067            assert!(
18068                failed
18069                    .body
18070                    .contains("No description of what this change does could be generated")
18071            );
18072            assert!(failed.body.is_ascii());
18073            assert!(f.runner.state.github_text.is_none());
18074            assert!(f.runner.state.change_summary.is_none());
18075            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18076            let evidence = crate::pr_outcome::summary_evidence(
18077                &f.runner.state,
18078                crate::pr_outcome::FinalHead::At(&head),
18079            );
18080            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18081            assert!(
18082                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18083                    .exists()
18084            );
18085            f.runner.state =
18086                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18087            std::fs::write(
18088                &f.reply,
18089                "TITLE: feat(own): add the own file\nAdds the committed own file.",
18090            )
18091            .unwrap();
18092            let retried = posted(&mut f).await;
18093            assert!(retried.body.contains("Adds the committed own file."));
18094            assert_eq!(
18095                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18096                "日本語の修正報告"
18097            );
18098        }
18099    }
18100
18101    #[tokio::test]
18102    async fn same_head_verification_and_findings_invalidate_public_prose() {
18103        let mut f = summary_fixture();
18104        posted(&mut f).await;
18105        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18106        let original = f
18107            .runner
18108            .state
18109            .change_summary
18110            .as_ref()
18111            .unwrap()
18112            .evidence
18113            .clone();
18114        ok_gate(&mut f.runner.state, Some(&head));
18115        std::fs::write(
18116            &f.reply,
18117            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18118        )
18119        .unwrap();
18120        let verified = posted(&mut f).await;
18121        assert!(verified.body.contains("now verified"));
18122        assert_ne!(
18123            f.runner.state.change_summary.as_ref().unwrap().evidence,
18124            original
18125        );
18126        let mut round = round_with_notes(1, Some("内部報告"));
18127        round.reviews.push(ReviewRecord {
18128            attempts: 0,
18129            reviewer: 1,
18130            agent: "alpha".into(),
18131            summary: "内部レビュー".into(),
18132            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18133            vote: None,
18134            failed: None,
18135            duration_ms: 0,
18136        });
18137        round
18138            .fix
18139            .as_mut()
18140            .unwrap()
18141            .rejected
18142            .push(crate::verdict::Rejection {
18143                id: "R1-1-1".into(),
18144                why: "呼び出し元から到達できない".into(),
18145            });
18146        f.runner.state.reviews = vec![round];
18147        // A description omitting the required ID is unusable, even with a title.
18148        let omitted = posted(&mut f).await;
18149        assert!(omitted.body.contains("No description"));
18150        assert!(omitted.body.contains("R1-1-1"));
18151        assert!(!omitted.body.contains("到達"));
18152        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18153        let described = posted(&mut f).await;
18154        assert!(described.body.contains("unreachable from callers"));
18155        assert!(described.body.contains("R1-1-1"));
18156        assert!(!described.body.contains("到達"));
18157        let evidence = f
18158            .runner
18159            .state
18160            .change_summary
18161            .as_ref()
18162            .unwrap()
18163            .evidence
18164            .clone();
18165        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18166        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18167        assert!(posted(&mut f).await.body.contains("rationale updated"));
18168        assert_ne!(
18169            f.runner.state.change_summary.as_ref().unwrap().evidence,
18170            evidence
18171        );
18172        // Old serialized summaries have no evidence fingerprint and must regenerate.
18173        f.runner
18174            .state
18175            .change_summary
18176            .as_mut()
18177            .unwrap()
18178            .evidence
18179            .clear();
18180        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18181        assert!(posted(&mut f).await.body.contains("current specification"));
18182    }
18183
18184    #[tokio::test]
18185    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18186        let mut f = summary_fixture();
18187        let m = posted(&mut f).await;
18188        assert_eq!(m.title, "feat(own): add the own file");
18189        assert!(
18190            m.body
18191                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
18192            "{}",
18193            m.body
18194        );
18195        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
18196        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
18197        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18198        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18199        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
18200        let kept = f.runner.state.change_summary.as_ref().unwrap();
18201        assert_eq!(
18202            (kept.head.as_str(), kept.base.as_str()),
18203            (head.as_str(), base.as_str())
18204        );
18205        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18206    }
18207
18208    #[tokio::test]
18209    async fn review_only_summary_receives_the_configured_github_language() {
18210        let mut f = summary_fixture();
18211        f.runner.state.config.graph.github_language = "ja".into();
18212        let captured = f.repo.join("summary-prompt.txt");
18213        let author = &mut f.runner.state.config.agents[0];
18214        author.command = vec![
18215            "sh".into(),
18216            "-c".into(),
18217            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18218        ];
18219        author.env.insert(
18220            "PROMPT_FILE".into(),
18221            captured.to_string_lossy().into_owned(),
18222        );
18223        std::fs::write(
18224            &f.reply,
18225            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18226        )
18227        .unwrap();
18228        let m = posted(&mut f).await;
18229        let prompt = std::fs::read_to_string(captured).unwrap();
18230        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18231        assert!(!prompt.contains("in English"));
18232        assert_eq!(m.title, "feat(own): ファイルを追加");
18233        assert!(m.body.contains("own.txt を追加します。"));
18234    }
18235
18236    #[tokio::test]
18237    async fn review_fixes_come_after_the_whole_change_summary() {
18238        let mut f = summary_fixture();
18239        f.runner.state.reviews = vec![
18240            round_with_notes(1, Some("handled the PATH case")),
18241            round_with_notes(2, None),
18242        ];
18243        let m = posted(&mut f).await;
18244        let summary = m.body.find("## Summary").expect("summary");
18245        let fixes = m.body.find("## Review rounds").expect("review rounds");
18246        assert!(summary < fixes, "{}", m.body);
18247        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18248        assert!(m.body.contains("## Final verification"), "{}", m.body);
18249    }
18250
18251    #[tokio::test]
18252    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18253        let mut f = summary_fixture();
18254        let first = posted(&mut f).await;
18255        let first_head = f.runner.state.change_summary.clone().unwrap().head;
18256        // The same head reuses the record: a different reply is not read.
18257        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18258        let again = posted(&mut f).await;
18259        assert_eq!((again.title, again.body), (first.title, first.body));
18260
18261        let g = |args: &[&str]| {
18262            let out = std::process::Command::new("git")
18263                .args(args)
18264                .current_dir(&f.repo)
18265                .quiet()
18266                .output()
18267                .expect("spawn git");
18268            assert!(out.status.success(), "git {args:?}");
18269        };
18270        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18271        g(&["add", "-A"]);
18272        g(&["commit", "-q", "-m", "fix(own): handle more"]);
18273        let second = posted(&mut f).await;
18274        let kept = f.runner.state.change_summary.clone().unwrap();
18275        assert_ne!(kept.head, first_head);
18276        assert!(second.body.contains("Not read yet."), "{}", second.body);
18277        assert!(!second.body.contains("so the first summary is read"));
18278        assert_eq!(second.title, "feat(own): other");
18279    }
18280
18281    #[tokio::test]
18282    async fn an_unavailable_summary_is_stated_and_never_invented() {
18283        let mut f = summary_fixture();
18284        std::fs::write(&f.reply, "no title line here").unwrap();
18285        let m = posted(&mut f).await;
18286        assert!(
18287            m.body
18288                .contains("No description of what this change does could be generated"),
18289            "{}",
18290            m.body
18291        );
18292        assert!(m.body.contains("internal reports remain in local records"));
18293        assert!(f.runner.state.change_summary.is_none());
18294        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18295        // A preview asks nobody and claims nothing.
18296        let winner = f.runner.state.candidates[0].clone();
18297        let preview = f
18298            .runner
18299            .guarded_pr_message(&winner, None, false)
18300            .await
18301            .unwrap()
18302            .unwrap();
18303        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18304    }
18305
18306    #[test]
18307    fn pr_message_review_single_commit_uses_its_subject() {
18308        let state = review_state(&["feat(nats): per-role user"]);
18309        let m = pr_message(&state, 'A');
18310        assert_eq!(m.title, "feat(nats): per-role user");
18311        assert!(!m.body.contains("Review the work already"), "{}", m.body);
18312        assert!(m.body.contains("## Commits under review"), "{}", m.body);
18313    }
18314
18315    #[test]
18316    fn pr_message_review_multi_commit_takes_the_oldest() {
18317        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18318        let m = pr_message(&state, 'A');
18319        assert_eq!(m.title, "feat: the change");
18320        for s in ["feat: the change", "fix: typo", "fix: again"] {
18321            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18322        }
18323    }
18324
18325    #[test]
18326    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18327        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18328            let mut state = review_state(&[first, "fix: later fixup"]);
18329            state.candidates[0].branch = "機能/ログイン".to_owned();
18330            let m = pr_message(&state, 'A');
18331            assert!(
18332                m.title.starts_with("chore: land candidate A of run"),
18333                "{}",
18334                m.title
18335            );
18336        }
18337    }
18338
18339    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18340        BranchFacts {
18341            commits: commits
18342                .iter()
18343                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18344                .collect(),
18345            stat: stat.to_owned(),
18346        }
18347    }
18348
18349    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18350        let mut r = review_round(true, 0, 1, 1, true, true);
18351        r.round = round;
18352        r.fix = notes.map(|n| FixRecord {
18353            agent: "fixer".to_owned(),
18354            addressed: Vec::new(),
18355            rejected: Vec::new(),
18356            notes: n.to_owned(),
18357            committed: true,
18358            failed: None,
18359            duration_ms: 0,
18360            continuation: None,
18361        });
18362        r
18363    }
18364
18365    #[test]
18366    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18367        let state = review_state(&["ignored"]);
18368        let f = facts(
18369            &[
18370                (
18371                    "fix(login): resolve PATH on macOS",
18372                    "Login shells skip rc files.",
18373                ),
18374                ("fix: address review", ""),
18375            ],
18376            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18377        );
18378        let m = pr_message_with(&state, 'A', Some(&f));
18379        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18380        assert!(
18381            m.body
18382                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
18383            "{}",
18384            m.body
18385        );
18386        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18387        assert!(m.body.contains("## Diff stat"), "{}", m.body);
18388        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18389        for banned in [
18390            "Review the work already",
18391            "no task statement",
18392            "Original task",
18393        ] {
18394            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18395        }
18396        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18397    }
18398
18399    #[test]
18400    fn pr_message_review_truncates_a_huge_first_commit_body() {
18401        let state = review_state(&["ignored"]);
18402        let f = facts(
18403            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18404            "s",
18405        );
18406        let m = pr_message_with(&state, 'A', Some(&f));
18407        assert!(m.body.len() < 30_000, "{}", m.body.len());
18408        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18409        assert!(
18410            m.body.contains("- ... 1 more commit(s)"),
18411            "{}",
18412            m.body.len()
18413        );
18414        assert!(m.body.ends_with("magi:candidate-a\n"));
18415    }
18416
18417    #[test]
18418    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18419        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18420        assert_eq!(m.title, "feat: x");
18421        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18422        assert!(!m.body.contains("Diff stat"), "{}", m.body);
18423        assert!(!m.body.contains("no task statement"), "{}", m.body);
18424    }
18425
18426    #[test]
18427    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18428        let mut state = review_state(&["日本語の件名"]);
18429        state.candidates[0].branch = "fix/macos-login-path".to_owned();
18430        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18431        state.candidates[0].branch = "機能/ログイン".to_owned();
18432        assert!(
18433            pr_message(&state, 'A')
18434                .title
18435                .starts_with("chore: land candidate A")
18436        );
18437    }
18438
18439    #[test]
18440    fn pr_message_review_fixes_survive_a_clean_final_round() {
18441        let mut state = review_state(&["feat: x"]);
18442        state.reviews = vec![
18443            round_with_notes(1, Some("handled the PATH case")),
18444            round_with_notes(2, None),
18445        ];
18446        let body = pr_message(&state, 'A').body;
18447        assert!(
18448            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18449            "{body}"
18450        );
18451        assert!(body.contains("### Round 1"), "{body}");
18452
18453        state.reviews = vec![
18454            round_with_notes(1, Some("first fix")),
18455            round_with_notes(2, Some("")),
18456            round_with_notes(3, Some("second fix")),
18457            round_with_notes(4, None),
18458        ];
18459        let body = pr_message(&state, 'A').body;
18460        assert!(
18461            body.contains("### Round 1") && !body.contains("first fix"),
18462            "{body}"
18463        );
18464        assert!(
18465            body.contains("### Round 3") && !body.contains("second fix"),
18466            "{body}"
18467        );
18468        assert!(body.contains("### Round 2"), "{body}");
18469    }
18470
18471    const FINAL: &str = "abcdef1234567890";
18472
18473    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18474        let mut r = round_with_notes(round, None);
18475        r.verified_head = verified.map(str::to_owned);
18476        r
18477    }
18478
18479    fn ok_gate(state: &mut RunState, head: Option<&str>) {
18480        state.gate = vec![CommandOutcome {
18481            command: "gate".to_owned(),
18482            code: Some(0),
18483            output_tail: String::new(),
18484            duration_ms: 0,
18485            resource_blocked: false,
18486        }];
18487        state.gate_ran = true;
18488        state.gate_head = head.map(str::to_owned);
18489    }
18490
18491    fn body_at(state: &RunState) -> String {
18492        pr_message_raw(
18493            state,
18494            'A',
18495            None,
18496            crate::pr_outcome::FinalHead::At(FINAL),
18497            ChangeNote::NotAsked,
18498        )
18499        .body
18500    }
18501
18502    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18503
18504    #[test]
18505    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18506        let mut state = review_state(&["feat: x"]);
18507        state.reviews = vec![
18508            round_with_notes(1, Some(SANDBOXED)),
18509            passing_round(2, Some(FINAL)),
18510        ];
18511        ok_gate(&mut state, Some(FINAL));
18512        let body = body_at(&state);
18513        assert!(!body.contains("Handled the PATH case."), "{body}");
18514        assert!(!body.contains("could not commit"), "{body}");
18515        assert!(!body.contains("could not run"), "{body}");
18516        assert!(
18517            body.contains("The fix is committed on the branch"),
18518            "{body}"
18519        );
18520        assert!(!body.contains("sandbox"), "{body}");
18521        assert!(
18522            body.contains("Tests (`verify.e2e`): passed on the final commit"),
18523            "{body}"
18524        );
18525        assert!(
18526            body.contains("Gate (`verify.gate`): passed on the final commit"),
18527            "{body}"
18528        );
18529        // The raw report is untouched in the record.
18530        assert!(
18531            state.reviews[0]
18532                .fix
18533                .as_ref()
18534                .unwrap()
18535                .notes
18536                .contains("could not commit")
18537        );
18538    }
18539
18540    #[test]
18541    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18542        let mut state = review_state(&["feat: x"]);
18543        let mut r1 = round_with_notes(1, Some(SANDBOXED));
18544        r1.fix.as_mut().unwrap().committed = false;
18545        let mut r2 = passing_round(2, Some(FINAL));
18546        r2.e2e[0].code = Some(1);
18547        state.reviews = vec![r1, r2];
18548        let body = body_at(&state);
18549        assert!(!body.contains("could not commit"), "{body}");
18550        assert!(!body.contains("could not run"), "{body}");
18551        assert!(body.contains("This round produced no commit"), "{body}");
18552        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18553        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18554        assert!(!body.contains("passed"), "{body}");
18555    }
18556
18557    #[test]
18558    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18559        let mut state = review_state(&["feat: x"]);
18560        state.reviews = vec![
18561            round_with_notes(1, Some(SANDBOXED)),
18562            passing_round(2, Some("0123456789abcdef")),
18563        ];
18564        ok_gate(&mut state, Some("0123456789abcdef"));
18565        let body = body_at(&state);
18566        assert!(!body.contains("HTTP tests could not run"), "{body}");
18567        // The commit claim is still settled by the structured record.
18568        assert!(!body.contains("could not commit"), "{body}");
18569        assert!(
18570            body.contains("passed on an earlier commit (`0123456`)"),
18571            "{body}"
18572        );
18573        assert!(!body.contains("passed on the final commit"), "{body}");
18574
18575        // A gate recorded before `gate_head` existed is "not recorded".
18576        ok_gate(&mut state, None);
18577        let body = body_at(&state);
18578        assert!(
18579            body.contains("passed on a commit that was not recorded"),
18580            "{body}"
18581        );
18582    }
18583
18584    #[test]
18585    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18586        let state = state_with_summary(
18587            "add retries to the client",
18588            "TITLE: feat: retries\n\nDid it.",
18589        );
18590        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18591        assert_eq!(m.title, "feat: retries");
18592        assert!(
18593            m.body.contains("<summary>Original task</summary>"),
18594            "{}",
18595            m.body
18596        );
18597        assert!(!m.body.contains("Commits under review"), "{}", m.body);
18598        assert!(
18599            m.body
18600                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18601            "{}",
18602            m.body
18603        );
18604    }
18605
18606    #[test]
18607    fn pr_message_review_bounds_a_long_english_subject() {
18608        let long = format!("feat: {}", "word ".repeat(100));
18609        let m = pr_message(&review_state(&[&long]), 'A');
18610        assert!(m.title.starts_with("feat: word"), "{}", m.title);
18611        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18612    }
18613
18614    #[test]
18615    fn pr_message_implementation_run_is_unchanged_by_review_support() {
18616        let state = state_with_summary("add retries\n\ndetails", "- did some things");
18617        let m = pr_message(&state, 'A');
18618        assert_eq!(m.title, "add retries");
18619        assert!(m.body.contains("<summary>Original task</summary>"));
18620        assert!(!m.body.contains("Commits under review"));
18621        assert_eq!(landing_subject_source(&state), state.instruction);
18622    }
18623
18624    #[test]
18625    fn review_run_squash_subject_is_the_change_not_the_prompt() {
18626        let state = review_state(&["feat: the change", "fix: typo"]);
18627        let source = landing_subject_source(&state);
18628        assert_eq!(land::merge_subject("", &source), "feat: the change");
18629        assert_eq!(
18630            land::merge_subject("magi: candidate A (uncommitted work)", &source),
18631            "feat: the change"
18632        );
18633        // An operator's rename still wins.
18634        assert_eq!(
18635            land::merge_subject("feat: renamed by hand", &source),
18636            "feat: renamed by hand"
18637        );
18638        let blank = review_state(&["日本語"]);
18639        assert!(
18640            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18641        );
18642    }
18643
18644    #[test]
18645    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18646        let state = review_state(&["feat: the change"]);
18647        let source = landing_subject_source(&state);
18648        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18649        assert_eq!(
18650            land::merge_subject(landing_title(&state, old), &source),
18651            "feat: the change"
18652        );
18653        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18654        let task = state_with_summary("add retries", "");
18655        assert_eq!(landing_title(&task, old), old);
18656    }
18657
18658    #[test]
18659    fn pr_message_describes_the_change_not_the_task() {
18660        let state = state_with_summary(
18661            "今回やってほしいこと: results projector を直す",
18662            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18663        );
18664        let m = pr_message(&state, 'A');
18665        assert_eq!(m.title, "fix(web): batch the runs list reads");
18666        assert!(
18667            m.body.starts_with("## Summary\n\n- reads run.json once"),
18668            "{}",
18669            m.body
18670        );
18671        assert!(!m.body.contains("TITLE:"), "{}", m.body);
18672        let task_at = m.body.find("今回やってほしいこと").unwrap();
18673        let details_at = m.body.find("<details>").unwrap();
18674        assert!(
18675            details_at < task_at,
18676            "the task lives inside <details>: {}",
18677            m.body
18678        );
18679        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18680        assert!(m.body.contains("magi:candidate-a"));
18681    }
18682
18683    #[test]
18684    fn pr_message_falls_back_to_the_task_without_a_title_line() {
18685        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18686        let m = pr_message(&state, 'A');
18687        assert_eq!(m.title, "add retries");
18688        assert!(
18689            m.body.contains("## Summary\n\n- did some things"),
18690            "{}",
18691            m.body
18692        );
18693
18694        let none = RunState::new(
18695            PathBuf::from("/repo"),
18696            "main".to_owned(),
18697            "abc1234".to_owned(),
18698            "add retries".to_owned(),
18699            Config::default(),
18700        );
18701        let m = pr_message(&none, 'A');
18702        assert_eq!(m.title, "add retries");
18703        assert!(!m.body.contains("## Summary"), "{}", m.body);
18704    }
18705
18706    #[test]
18707    fn pr_message_refuses_the_candidate_commit_subject() {
18708        for bad in [
18709            "TITLE: magi: candidate A (uncommitted work)",
18710            "TITLE: chore: stuff (uncommitted work)",
18711            "TITLE:   ",
18712        ] {
18713            let state = state_with_summary("add retries", bad);
18714            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18715        }
18716    }
18717
18718    #[test]
18719    fn pr_message_bounds_a_very_long_task_and_title() {
18720        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18721        let state = state_with_summary(&long, "- nothing");
18722        let m = pr_message(&state, 'A');
18723        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18724        assert!(!m.title.contains('\n'));
18725
18726        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18727        let m = pr_message(&state, 'A');
18728        assert!(m.title.starts_with("feat: "));
18729        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18730        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18731    }
18732
18733    fn long_title_of(instruction: &str) -> String {
18734        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18735    }
18736
18737    #[test]
18738    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18739        let first = "Make the landing path keep a readable title for long tasks";
18740        let line = format!(
18741            "{first}. {}",
18742            "And then keep going with more words ".repeat(20)
18743        );
18744        let t = long_title_of(&line);
18745        assert_eq!(t, first);
18746        assert!(!t.starts_with("chore: land"));
18747    }
18748
18749    #[test]
18750    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18751        let line = "word ".repeat(200);
18752        let t = long_title_of(&line);
18753        assert!(t.ends_with("word..."), "{t}");
18754        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18755    }
18756
18757    #[test]
18758    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18759        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18760            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18761        }
18762    }
18763
18764    #[test]
18765    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18766        let mut state = state_with_summary("再試行を追加する", "- nothing");
18767        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18768        state.config.graph.github_language = "ja".into();
18769        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18770    }
18771
18772    #[test]
18773    fn pr_message_title_limit_is_exact() {
18774        let at = "a".repeat(PR_TITLE_MAX);
18775        assert_eq!(long_title_of(&at), at);
18776        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18777        assert!(over.ends_with("..."), "{over}");
18778        assert_eq!(over.chars().count(), PR_TITLE_MAX);
18779    }
18780
18781    #[test]
18782    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18783        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18784        let t = long_title_of(&line);
18785        assert!(t.ends_with("..."), "{t}");
18786        assert!(t.is_ascii(), "{t}");
18787    }
18788
18789    #[test]
18790    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18791        let line = format!(
18792            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18793            "plus more filler words ".repeat(20)
18794        );
18795        let t = long_title_of(&line);
18796        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18797    }
18798
18799    #[test]
18800    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18801        let line = format!("\n\n# {}", "title words ".repeat(40));
18802        let state = state_with_summary(&line, "- nothing");
18803        let m = pr_message(&state, 'A');
18804        assert!(m.title.starts_with("title words"), "{}", m.title);
18805        assert_eq!(
18806            land::merge_subject(&m.title, &landing_subject_source(&state)),
18807            m.title
18808        );
18809        // An operator's rename wins untouched.
18810        assert_eq!(
18811            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18812            "feat: renamed by hand"
18813        );
18814    }
18815
18816    #[tokio::test]
18817    async fn github_text_rewrite_is_bounded_and_falls_back() {
18818        crate::run::pin_test_home();
18819        for (reply, accepted) in [
18820            (
18821                "TITLE: fix: retries\nAdd retries for failed requests.",
18822                true,
18823            ),
18824            (
18825                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18826                false,
18827            ),
18828            (
18829                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18830                true,
18831            ),
18832            (
18833                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18834                true,
18835            ),
18836            (
18837                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18838                true,
18839            ),
18840            (
18841                "TITLE: fix: retries\n日本語の説明です。 token=secret",
18842                false,
18843            ),
18844            ("TITLE: fix: retries\n", false),
18845        ] {
18846            let dir = tempfile::tempdir().unwrap();
18847            let mut runner = runner_at(RunStatus::Gating);
18848            runner.state = state_with_summary(
18849                "add retries",
18850                "TITLE: fix: retries\n日本語の説明を書きます。",
18851            );
18852            runner.state.repo = dir.path().to_owned();
18853            runner.state.candidates[0].worktree = dir.path().to_owned();
18854            let mut author = spec("alpha");
18855            author.command = vec![
18856                "sh".into(),
18857                "-c".into(),
18858                "printf '%s' \"$REWRITE_REPLY\"".into(),
18859            ];
18860            author.env.insert("REWRITE_REPLY".into(), reply.into());
18861            runner.state.config.agents = vec![author];
18862            seed_public_summary(&mut runner);
18863            let winner = runner.state.candidates[0].clone();
18864            let first = runner
18865                .guarded_pr_message(&winner, None, true)
18866                .await
18867                .unwrap();
18868            let message = if accepted {
18869                first.expect("a passing rewrite never asks")
18870            } else {
18871                // Withheld: the run parks on a question and posts nothing yet.
18872                assert!(first.is_none());
18873                assert!(runner.state.parked);
18874                let store = ask::Questions::open();
18875                let asked: Vec<_> = store
18876                    .list()
18877                    .into_iter()
18878                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18879                    .collect();
18880                assert_eq!(asked.len(), 1);
18881                assert!(!asked[0].detail.contains("secret"));
18882                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18883                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18884                assert_eq!(
18885                    asked[0]
18886                        .choices
18887                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
18888                    rewrite.is_some()
18889                );
18890                // Silence: the question lapses and the neutral text is used,
18891                // without a second rewrite call or a second question.
18892                store
18893                    .update(&asked[0].id, |q| {
18894                        q.abandon("test");
18895                        Ok(())
18896                    })
18897                    .unwrap();
18898                let again = runner
18899                    .guarded_pr_message(&winner, None, true)
18900                    .await
18901                    .unwrap()
18902                    .expect("an abandoned question falls back");
18903                let n = store
18904                    .list()
18905                    .into_iter()
18906                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18907                    .count();
18908                assert_eq!(n, 1, "asked once per rejected text");
18909                again
18910            };
18911            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18912            assert!(!message.body.contains("secret"));
18913            assert!(!message.body.contains("prefix"));
18914            assert!(!message.title.contains("secret"));
18915            assert_eq!(
18916                message.body.contains("Add retries for failed requests."),
18917                accepted
18918            );
18919            assert_eq!(
18920                runner.state.seats["github-text-rewrite"].turns, 1,
18921                "only one rewrite invocation"
18922            );
18923            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18924            if accepted {
18925                assert!(runner.state.github_text.is_none());
18926                assert!(!runner.state.parked);
18927                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18928                assert!(message.body.contains("## Final verification"));
18929                assert!(message.body.contains("Evidence: final diff"));
18930                assert!(
18931                    message
18932                        .body
18933                        .contains(&format!("magi:run/{}", runner.state.id))
18934                );
18935            }
18936            if !accepted {
18937                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18938                assert!(
18939                    message
18940                        .body
18941                        .contains(&format!("magi:run/{}", runner.state.id))
18942                );
18943            }
18944        }
18945    }
18946
18947    fn scripted(id: &str, reply: &str) -> AgentSpec {
18948        let mut a = spec(id);
18949        a.command = vec![
18950            "sh".into(),
18951            "-c".into(),
18952            "printf '%s' \"$REWRITE_REPLY\"".into(),
18953        ];
18954        a.env.insert("REWRITE_REPLY".into(), reply.into());
18955        a
18956    }
18957
18958    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18959    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18960
18961    /// Gate tests inject generated public output, not internal reports.
18962    fn seed_public_summary(runner: &mut Runner) {
18963        let repo = &runner.state.repo;
18964        let g = |args: &[&str]| {
18965            let out = std::process::Command::new("git")
18966                .args(args)
18967                .current_dir(repo)
18968                .quiet()
18969                .output()
18970                .unwrap();
18971            assert!(out.status.success(), "git {args:?}");
18972            String::from_utf8_lossy(&out.stdout).trim().to_owned()
18973        };
18974        g(&["init", "-q", "-b", "main"]);
18975        g(&[
18976            "-c",
18977            "user.name=Test",
18978            "-c",
18979            "user.email=test@example.invalid",
18980            "commit",
18981            "-q",
18982            "--allow-empty",
18983            "-m",
18984            "feat: fixture",
18985        ]);
18986        let head = g(&["rev-parse", "HEAD"]);
18987        g(&["update-ref", "refs/remotes/origin/main", &head]);
18988        let evidence = crate::pr_outcome::summary_evidence(
18989            &runner.state,
18990            crate::pr_outcome::FinalHead::At(&head),
18991        );
18992        let generated = &runner.state.candidates[0].summary;
18993        runner.state.change_summary = Some(crate::run::ChangeSummary {
18994            base: head.clone(),
18995            head,
18996            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18997            title: summary_title(generated).unwrap(),
18998            body: summary_without_title(generated),
18999        });
19000    }
19001
19002    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19003        let mut runner = runner_at(RunStatus::Gating);
19004        runner.state = state_with_summary("add retries", JA_SUMMARY);
19005        if review_only {
19006            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19007        }
19008        runner.state.repo = dir.to_owned();
19009        runner.state.candidates[0].worktree = dir.to_owned();
19010        runner.state.config.agents = agents;
19011        seed_public_summary(&mut runner);
19012        runner
19013    }
19014
19015    fn gate_questions(runner: &Runner) -> usize {
19016        ask::Questions::open()
19017            .list()
19018            .into_iter()
19019            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19020            .count()
19021    }
19022
19023    #[tokio::test]
19024    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19025        crate::run::pin_test_home();
19026        let dir = tempfile::tempdir().unwrap();
19027        let mut runner = rewrite_runner(
19028            dir.path(),
19029            true,
19030            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19031        );
19032        runner.state.config.roles.language_judge =
19033            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19034        let winner = runner.state.candidates[0].clone();
19035        let msg = runner
19036            .guarded_pr_message(&winner, None, true)
19037            .await
19038            .unwrap()
19039            .expect("an English rewrite posts without asking");
19040        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19041        assert!(msg.body.contains("<details>kept</details>"));
19042        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19043        assert_eq!(gate_questions(&runner), 0);
19044        assert!(runner.state.github_text.is_none());
19045        assert!(!runner.state.seats.contains_key("impl-A"));
19046        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19047    }
19048
19049    #[tokio::test]
19050    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19051        crate::run::pin_test_home();
19052        let dir = tempfile::tempdir().unwrap();
19053        let mut runner = rewrite_runner(
19054            dir.path(),
19055            false,
19056            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19057        );
19058        runner.state.config.roles.language_judge =
19059            Some(crate::config::AgentChoice::One("beta".into()));
19060        runner.state.seats.insert(
19061            "impl-A".into(),
19062            SeatState::new("impl-A", "alpha", runner.state.seed),
19063        );
19064        let winner = runner.state.candidates[0].clone();
19065        let msg = runner
19066            .guarded_pr_message(&winner, None, true)
19067            .await
19068            .unwrap()
19069            .expect("rewritten");
19070        assert!(msg.body.contains("<details>kept</details>"));
19071        assert_eq!(runner.state.seats["impl-A"].turns, 0);
19072        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19073        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19074    }
19075
19076    #[tokio::test]
19077    async fn rewrite_reuses_a_resumable_implementer_session() {
19078        crate::run::pin_test_home();
19079        let dir = tempfile::tempdir().unwrap();
19080        let mut runner = rewrite_runner(
19081            dir.path(),
19082            false,
19083            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19084        );
19085        runner.state.config.roles.language_judge =
19086            Some(crate::config::AgentChoice::One("beta".into()));
19087        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19088        seat.turns = 1;
19089        runner.state.seats.insert("impl-A".into(), seat);
19090        let winner = runner.state.candidates[0].clone();
19091        runner
19092            .guarded_pr_message(&winner, None, true)
19093            .await
19094            .unwrap()
19095            .expect("rewritten");
19096        assert_eq!(runner.state.seats["impl-A"].turns, 2);
19097        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19098    }
19099
19100    #[tokio::test]
19101    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19102        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
19103        // session, was out of quota, and nothing else was asked.
19104        crate::run::pin_test_home();
19105        let dir = tempfile::tempdir().unwrap();
19106        let mut exhausted = spec("alpha");
19107        exhausted.command = vec!["false".into()];
19108        let mut runner = rewrite_runner(
19109            dir.path(),
19110            false,
19111            vec![exhausted, scripted("beta", EN_REWRITE)],
19112        );
19113        // The implementer's own agent is in the role too: it is not asked twice.
19114        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19115            "alpha".into(),
19116            "beta".into(),
19117        ]));
19118        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19119        seat.turns = 1;
19120        runner.state.seats.insert("impl-A".into(), seat);
19121        let winner = runner.state.candidates[0].clone();
19122        let msg = runner
19123            .guarded_pr_message(&winner, None, true)
19124            .await
19125            .unwrap()
19126            .expect("the role rewrites after the implementer fails");
19127        assert!(msg.body.contains("`src/net.rs`"));
19128        assert_eq!(
19129            runner.state.seats["impl-A"].turns, 2,
19130            "the implementer was tried once"
19131        );
19132        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19133        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19134    }
19135
19136    #[tokio::test]
19137    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19138        crate::run::pin_test_home();
19139        let dir = tempfile::tempdir().unwrap();
19140        let mut broken = spec("bad");
19141        broken.command = vec!["false".into()];
19142        let mut runner = rewrite_runner(
19143            dir.path(),
19144            true,
19145            vec![
19146                broken,
19147                scripted("noformat", "Add retries."),
19148                scripted("good", EN_REWRITE),
19149            ],
19150        );
19151        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19152            "bad".into(),
19153            "noformat".into(),
19154            "good".into(),
19155        ]));
19156        let winner = runner.state.candidates[0].clone();
19157        let msg = runner
19158            .guarded_pr_message(&winner, None, true)
19159            .await
19160            .unwrap()
19161            .expect("the third agent rewrites");
19162        assert!(msg.body.contains("`src/net.rs`"));
19163        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19164        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19165    }
19166
19167    #[tokio::test]
19168    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19169        crate::run::pin_test_home();
19170        // (agents, role): still Japanese; no TITLE; nothing resolvable.
19171        for (agents, role, saved) in [
19172            (
19173                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19174                Some("beta"),
19175                true,
19176            ),
19177            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19178            (Vec::new(), None, false),
19179        ] {
19180            let dir = tempfile::tempdir().unwrap();
19181            let mut runner = rewrite_runner(dir.path(), true, agents);
19182            runner.state.config.roles.language_judge =
19183                role.map(|r| crate::config::AgentChoice::One(r.into()));
19184            let winner = runner.state.candidates[0].clone();
19185            let first = runner
19186                .guarded_pr_message(&winner, None, true)
19187                .await
19188                .unwrap();
19189            assert!(first.is_none(), "withheld text parks the run");
19190            assert!(runner.state.parked);
19191            assert_eq!(gate_questions(&runner), 1);
19192            let g = runner.state.github_text.as_ref().unwrap();
19193            assert_eq!(g.rewrite.is_some(), saved);
19194            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
19195        }
19196    }
19197
19198    #[tokio::test]
19199    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
19200        crate::run::pin_test_home();
19201        let dir = tempfile::tempdir().unwrap();
19202        let mut runner = runner_at(RunStatus::Gating);
19203        runner.state =
19204            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19205        runner.state.repo = dir.path().to_owned();
19206        runner.state.candidates[0].worktree = dir.path().to_owned();
19207        let mut author = spec("alpha");
19208        author.command = vec![
19209            "sh".into(),
19210            "-c".into(),
19211            "printf '%s' \"$REWRITE_REPLY\"".into(),
19212        ];
19213        author.env.insert(
19214            "REWRITE_REPLY".into(),
19215            format!(
19216                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19217                "English sentence. ".repeat(600)
19218            ),
19219        );
19220        runner.state.config.agents = vec![author];
19221        seed_public_summary(&mut runner);
19222        let winner = runner.state.candidates[0].clone();
19223        assert!(
19224            runner
19225                .guarded_pr_message(&winner, None, true)
19226                .await
19227                .unwrap()
19228                .is_none()
19229        );
19230        let record = runner.state.github_text.clone().unwrap();
19231        let rewrite = record.rewrite.clone().unwrap();
19232        assert_eq!(rewrite.failed, ["body-language"]);
19233        assert!(
19234            rewrite
19235                .body
19236                .ends_with(&format!("magi:run/{}", runner.state.id))
19237        );
19238        let store = ask::Questions::open();
19239        let q = store.get(record.question.as_ref().unwrap()).unwrap();
19240        assert!(
19241            q.choices
19242                .contains(&crate::github_text::USE_REWRITE.to_owned())
19243        );
19244        for label in [
19245            "ORIGINAL",
19246            "FALLBACK",
19247            "REWRITE",
19248            "Rules still failed: body-language",
19249            "truncated",
19250        ] {
19251            assert!(q.detail.contains(label), "{label}");
19252        }
19253        let artifact = std::fs::read_to_string(crate::run::artifact_path(
19254            &runner.state,
19255            &crate::github_text::artifact_name(&record.fingerprint),
19256        ))
19257        .unwrap();
19258        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19259        // Restart from serialized state, then retry after a bad owner title.
19260        runner.state =
19261            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19262        store
19263            .update(&q.id, |q| {
19264                q.say("まだ日本語")?;
19265                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19266            })
19267            .unwrap();
19268        assert!(
19269            runner
19270                .guarded_pr_message(&winner, None, true)
19271                .await
19272                .unwrap()
19273                .is_none()
19274        );
19275        let retry = runner.state.github_text.as_ref().unwrap();
19276        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19277        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19278        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19279        assert!(
19280            q.posting_comparison
19281                .as_ref()
19282                .unwrap()
19283                .matches_detail(&q.detail)
19284        );
19285        store
19286            .update(&q.id, |q| {
19287                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19288            })
19289            .unwrap();
19290        let message = runner
19291            .guarded_pr_message(&winner, None, true)
19292            .await
19293            .unwrap()
19294            .unwrap();
19295        assert_eq!(message.title, rewrite.title);
19296        assert_eq!(message.body, rewrite.body);
19297        assert_eq!(
19298            runner.state.github_text.as_ref().unwrap().choice,
19299            Some(GithubTextChoice::Rewrite)
19300        );
19301        let again = runner
19302            .guarded_pr_message(&winner, None, true)
19303            .await
19304            .unwrap()
19305            .unwrap();
19306        assert_eq!(again.title, message.title);
19307        assert_eq!(again.body, message.body);
19308        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19309        assert!(
19310            !runner
19311                .state
19312                .events
19313                .iter()
19314                .any(|e| e.message.contains(&rewrite.title)
19315                    || e.message.contains("English sentence."))
19316        );
19317    }
19318
19319    #[tokio::test]
19320    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19321        crate::run::pin_test_home();
19322        let dir = tempfile::tempdir().unwrap();
19323        let mut runner = runner_at(RunStatus::Gating);
19324        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19325        runner.state.repo = dir.path().to_owned();
19326        runner.state.candidates[0].worktree = dir.path().to_owned();
19327        seed_public_summary(&mut runner);
19328        let winner = runner.state.candidates[0].clone();
19329        assert!(
19330            runner
19331                .guarded_pr_message(&winner, None, true)
19332                .await
19333                .unwrap()
19334                .is_none()
19335        );
19336        let record = runner.state.github_text.as_ref().unwrap();
19337        assert!(record.rewrite.is_none());
19338        // Records written before the additive field deserialize with no rewrite.
19339        let mut old = serde_json::to_value(record).unwrap();
19340        old.as_object_mut().unwrap().remove("rewrite");
19341        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19342        assert!(old.rewrite.is_none());
19343        let id = record.question.clone().unwrap();
19344        ask::Questions::open()
19345            .update(&id, |q| {
19346                q.choices.push(crate::github_text::USE_REWRITE.into());
19347                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19348            })
19349            .unwrap();
19350        let message = runner
19351            .guarded_pr_message(&winner, None, true)
19352            .await
19353            .unwrap()
19354            .unwrap();
19355        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19356        assert_eq!(
19357            runner.state.github_text.as_ref().unwrap().choice,
19358            Some(GithubTextChoice::Fallback)
19359        );
19360    }
19361
19362    #[tokio::test]
19363    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19364        crate::run::pin_test_home();
19365        let dir = tempfile::tempdir().unwrap();
19366        let mut runner = runner_at(RunStatus::Gating);
19367        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19368        runner.state.repo = dir.path().to_owned();
19369        runner.state.candidates[0].worktree = dir.path().to_owned();
19370        seed_public_summary(&mut runner);
19371        let winner = runner.state.candidates[0].clone();
19372        assert!(
19373            runner
19374                .guarded_pr_message(&winner, None, true)
19375                .await
19376                .unwrap()
19377                .is_none()
19378        );
19379        let store = ask::Questions::open();
19380        let find = |run: &str| {
19381            store
19382                .list()
19383                .into_iter()
19384                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19385                .unwrap()
19386        };
19387        let run = runner.state.id.clone();
19388        let q = find(&run);
19389        let fp = runner
19390            .state
19391            .github_text
19392            .as_ref()
19393            .unwrap()
19394            .fingerprint
19395            .clone();
19396        let name = crate::github_text::artifact_name(&fp);
19397        let path = crate::run::artifact_path(&runner.state, &name);
19398        assert!(q.detail.contains(&path.display().to_string()));
19399        assert!(q.detail.contains("title-language: title"));
19400        let saved = std::fs::read_to_string(&path).expect("artifact written");
19401        assert!(saved.contains("日本語のタイトル"));
19402        assert!(q.cwd.is_none());
19403        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19404        // A bad replacement is refused and asked about once more.
19405        let say = |id: &str, text: &str, choice: &str| {
19406            store
19407                .update(id, |q| {
19408                    q.thread.push(ask::Turn {
19409                        who: ask::Who::Operator,
19410                        body: text.to_owned(),
19411                        at: jiff::Timestamp::now(),
19412                        note: None,
19413                    });
19414                    q.answer(ask::Answer::Choice(choice.to_owned()))
19415                })
19416                .unwrap();
19417        };
19418        say(&q.id, "まだ日本語", "use my text");
19419        assert!(
19420            runner
19421                .guarded_pr_message(&winner, None, true)
19422                .await
19423                .unwrap()
19424                .is_none()
19425        );
19426        let second = find(&run);
19427        assert_ne!(second.id, q.id);
19428        assert!(second.detail.contains("did not pass"));
19429        // The retry still compares the original text; the rejected
19430        // replacement is named by category only.
19431        assert!(second.detail.contains("日本語のタイトル"));
19432        assert!(!second.detail.contains("まだ日本語"));
19433        assert!(second.detail.contains(&path.display().to_string()));
19434        say(&second.id, "fix: retry failed requests", "use my text");
19435        let message = runner
19436            .guarded_pr_message(&winner, None, true)
19437            .await
19438            .unwrap()
19439            .expect("a vetted replacement is posted");
19440        assert_eq!(message.title, "fix: retry failed requests");
19441        assert_eq!(
19442            runner
19443                .state
19444                .github_text
19445                .as_ref()
19446                .unwrap()
19447                .chosen_title
19448                .as_deref(),
19449            Some("fix: retry failed requests")
19450        );
19451        assert!(runner.state.github_text.as_ref().unwrap().resolved);
19452        // Resumed again: the saved decision is applied, nothing is re-asked.
19453        let again = runner
19454            .guarded_pr_message(&winner, None, true)
19455            .await
19456            .unwrap()
19457            .unwrap();
19458        assert_eq!(again.title, message.title);
19459        assert_eq!(
19460            store
19461                .list()
19462                .into_iter()
19463                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19464                .count(),
19465            2
19466        );
19467    }
19468
19469    #[test]
19470    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19471        // What magi itself writes stays English under any configured language,
19472        // so a future localisation of these headings fails here. (The agents'
19473        // own text is also checked by the posting gate.)
19474        let mut state = state_with_summary(
19475            "add retries",
19476            "TITLE: fix(web): batch reads\n- reads run.json once",
19477        );
19478        state.config.graph.language = "ja".to_owned();
19479        let m = pr_message(&state, 'A');
19480        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19481        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19482
19483        // The task is the operator's own text: it goes in untouched, and the
19484        // fallback title (no summary) may be in its language too.
19485        let task = "今回やってほしいこと: results projector を直す";
19486        let mut state = state_with_summary(task, "- no title line");
19487        state.config.graph.language = "ja".to_owned();
19488        let m = pr_message(&state, 'A');
19489        assert_eq!(
19490            m.title,
19491            format!("chore: land candidate A of run {}", state.id)
19492        );
19493        assert!(
19494            m.body.contains(&format!(
19495                "<summary>Original task</summary>\n\n{task}\n\n</details>"
19496            )),
19497            "{}",
19498            m.body
19499        );
19500    }
19501
19502    #[test]
19503    fn pr_message_scrubs_home_paths_and_addresses() {
19504        let state = state_with_summary(
19505            "fix it in /Users/someone/src/x",
19506            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19507        );
19508        let m = pr_message(&state, 'A');
19509        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19510            assert!(!m.body.contains(leak), "{}", m.body);
19511        }
19512        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19513    }
19514
19515    #[test]
19516    fn pr_message_survives_a_task_that_closes_details() {
19517        let state = state_with_summary("a </details> b", "TITLE: fix: x");
19518        let m = pr_message(&state, 'A');
19519        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19520    }
19521
19522    #[test]
19523    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19524        let cmd = manual_merge_command(
19525            MergeStyle::Squash,
19526            Path::new("/repo"),
19527            "b",
19528            "fix: \"quoted\" $(x) `y`\n\nbody",
19529        );
19530        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19531    }
19532
19533    #[test]
19534    fn manual_merge_command_matches_the_configured_style() {
19535        let repo = Path::new("/repo");
19536        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19537
19538        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19539        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19540
19541        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19542        assert_eq!(
19543            squash,
19544            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19545             \"Merge magi run 0832 (candidate A)\""
19546        );
19547
19548        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19549        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19550    }
19551
19552    #[test]
19553    fn a_nudge_gets_a_quarter_of_the_budget() {
19554        // The judge and implement budgets magi ships with.
19555        assert_eq!(retry_budget(secs(1200), true), secs(300));
19556        assert_eq!(retry_budget(secs(3600), true), secs(900));
19557    }
19558
19559    #[test]
19560    fn a_resent_prompt_keeps_the_whole_budget() {
19561        // The seat kept no context, so the retry is the original job again and
19562        // shortening it would only guarantee a second failure.
19563        assert_eq!(retry_budget(secs(1200), false), secs(1200));
19564        assert_eq!(retry_budget(secs(60), false), secs(60));
19565    }
19566
19567    #[test]
19568    fn the_floor_never_exceeds_the_original_budget() {
19569        // A short configured timeout must not be *raised* by the floor: the
19570        // operator asked for a bound, and a retry may not outlast the attempt
19571        // it is retrying.
19572        assert_eq!(retry_budget(secs(60), true), secs(60));
19573        assert_eq!(retry_budget(secs(480), true), secs(120));
19574        assert_eq!(retry_budget(secs(0), true), secs(0));
19575    }
19576
19577    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19578        agent::CommandEvidence {
19579            id: "item1".to_owned(),
19580            description: "cargo test".to_owned(),
19581            exit_code,
19582            result_summary: String::new(),
19583            source: "codex".to_owned(),
19584        }
19585    }
19586
19587    #[test]
19588    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19589        // No evidence is not the same fact as unconfirmed evidence: a
19590        // backend with no adapter, or a reply that ran no commands at all,
19591        // must not be misread as carrying a dangling job.
19592        assert!(!has_unconfirmed_command(&[]));
19593    }
19594
19595    #[test]
19596    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19597        // Deliberately not a check on the exit code's *value*: a fixer
19598        // legitimately runs something that fails mid-iteration before it
19599        // succeeds, and that must never by itself reopen a valid report.
19600        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19601        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19602        assert!(!has_unconfirmed_command(&[
19603            evidence(Some(0)),
19604            evidence(Some(101))
19605        ]));
19606    }
19607
19608    #[test]
19609    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19610        assert!(has_unconfirmed_command(&[
19611            evidence(Some(0)),
19612            evidence(None)
19613        ]));
19614    }
19615
19616    #[test]
19617    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19618        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19619        assert_eq!(
19620            verified_noop_claim(true, &[], text).as_deref(),
19621            Some("already fixed by b32cfc4, on main.")
19622        );
19623    }
19624
19625    #[test]
19626    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19627        // A timeout or a bad exit code reads as the ordinary loss it is,
19628        // whatever the reply's own prose claims.
19629        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19630        assert!(verified_noop_claim(false, &[], text).is_none());
19631    }
19632
19633    #[test]
19634    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19635        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19636        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19637        // A confirmed command alongside the marker is fine.
19638        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19639    }
19640
19641    #[test]
19642    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19643        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19644    }
19645
19646    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
19647    /// pair, in order, labelled A, B, C, ...
19648    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19649        runner.state.candidates = shape
19650            .iter()
19651            .enumerate()
19652            .map(|(i, &(empty, verified))| Candidate {
19653                index: i,
19654                label: (b'A' + i as u8) as char,
19655                agent: "sonnet".to_owned(),
19656                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19657                worktree: PathBuf::from(format!("/wt/{i}")),
19658                summary: String::new(),
19659                stat: String::new(),
19660                files: 0,
19661                commits: 0,
19662                empty,
19663                failed: None,
19664                verified_noop: verified.map(str::to_owned),
19665                duration_ms: 0,
19666                folded: false,
19667            })
19668            .collect();
19669    }
19670
19671    #[test]
19672    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19673        ask_test_home();
19674        let mut runner = runner_at(RunStatus::Implementing);
19675        set_candidates(
19676            &mut runner,
19677            &[
19678                (true, Some("already on main at b32cfc4")),
19679                (true, Some("same fix, see the existing test")),
19680            ],
19681        );
19682
19683        runner
19684            .after_implement()
19685            .expect("a verified no-op is not an error");
19686
19687        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19688    }
19689
19690    #[test]
19691    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19692        ask_test_home();
19693        let mut runner = runner_at(RunStatus::Implementing);
19694        // Candidate A declares a verified no-op; candidate B simply wrote
19695        // nothing and said nothing about why. One candidate's claim is not
19696        // the whole run's agreement.
19697        set_candidates(
19698            &mut runner,
19699            &[(true, Some("already on main at b32cfc4")), (true, None)],
19700        );
19701
19702        let err = runner
19703            .after_implement()
19704            .expect_err("an unverified empty candidate must still fail the run");
19705
19706        assert!(
19707            err.to_string().contains("no candidate produced a change"),
19708            "{err}"
19709        );
19710        assert_eq!(runner.state.status, RunStatus::Failed);
19711    }
19712
19713    #[test]
19714    fn after_implement_still_fails_an_ordinary_all_empty_run() {
19715        ask_test_home();
19716        let mut runner = runner_at(RunStatus::Implementing);
19717        set_candidates(&mut runner, &[(true, None), (true, None)]);
19718
19719        let err = runner
19720            .after_implement()
19721            .expect_err("no candidate declared anything; this is an ordinary failure");
19722
19723        assert!(
19724            err.to_string().contains("no candidate produced a change"),
19725            "{err}"
19726        );
19727        assert_eq!(runner.state.status, RunStatus::Failed);
19728    }
19729}
19730
19731/// A graft rollback whose restore could not be confirmed (see
19732/// [`Runner::graft`]).
19733#[derive(Debug)]
19734struct GraftRestoreFailed(String);
19735
19736impl std::fmt::Display for GraftRestoreFailed {
19737    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19738        write!(
19739            f,
19740            "could not restore the winner to its pre-graft commit: {}",
19741            self.0
19742        )
19743    }
19744}
19745
19746impl std::error::Error for GraftRestoreFailed {}
19747
19748/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
19749/// not the agent: they never draw on an agent's own timeout, and running out of
19750/// them never reads as an agent failure.
19751#[cfg(test)]
19752mod resource_wait_tests {
19753    use super::*;
19754    use std::collections::BTreeMap;
19755
19756    fn sh(id: &str, script: &str) -> AgentSpec {
19757        AgentSpec {
19758            id: id.to_owned(),
19759            kind: crate::config::AgentKind::Command,
19760            model: None,
19761            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19762            extra_args: Vec::new(),
19763            env: BTreeMap::new(),
19764            prompt_delivery: None,
19765            max_turn_secs: None,
19766        }
19767    }
19768
19769    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19770        SeatJob {
19771            seat: SeatState::new("impl-A", &spec.id, 1),
19772            spec,
19773            cwd: dir.to_path_buf(),
19774            prompt: "do it".to_owned(),
19775            timeout,
19776            allow_write: true,
19777            sessions: false,
19778            artifacts: dir.join("art"),
19779            stem: "impl-A".to_owned(),
19780            handover: None,
19781        }
19782    }
19783
19784    fn state(resource_wait_secs: u64) -> RunState {
19785        let mut config = Config::default();
19786        config.graph.timeout_resource_wait = resource_wait_secs;
19787        RunState::new(
19788            PathBuf::from("/repo"),
19789            "main".to_owned(),
19790            "abc1234".to_owned(),
19791            "task".to_owned(),
19792            config,
19793        )
19794    }
19795
19796    fn hold(cache: &Path) -> crate::cache::Guard {
19797        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19798        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19799            crate::cache::AcquireOutcome::Acquired(g) => g,
19800            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19801        }
19802    }
19803
19804    async fn run(
19805        jobs: Vec<SeatJob>,
19806        sem: Arc<Semaphore>,
19807        cache: Option<&Path>,
19808        st: &mut RunState,
19809    ) -> Vec<(usize, SeatState, AgentOutcome)> {
19810        let prompts = Prompts::default();
19811        let ctx = WaveCtx {
19812            run: "r",
19813            node: "implement",
19814            prompts: &prompts,
19815            cache,
19816            round: None,
19817            carry_seats: false,
19818            pauses: None,
19819        };
19820        wave(jobs, sem, &ctx, st, 0).await
19821    }
19822
19823    #[tokio::test]
19824    async fn verification_cache_wait_preserves_the_command_budget() {
19825        crate::run::pin_test_home();
19826        let tmp = tempfile::tempdir().unwrap();
19827        let cache = tmp.path().join("target");
19828        let guard = hold(&cache);
19829        let release = tokio::spawn(async move {
19830            tokio::time::sleep(Duration::from_millis(100)).await;
19831            guard.release();
19832        });
19833        let mut st = state(10);
19834        let budget = Duration::from_millis(50);
19835        let wait = repository_wait(&st, None);
19836        let (outcomes, _) = with_cache_lease(
19837            &mut st,
19838            Some(&cache),
19839            "gate",
19840            "gate",
19841            tmp.path(),
19842            "abc1234",
19843            budget,
19844            "test gate",
19845            &wait,
19846            |_, available| async move {
19847                assert_eq!(
19848                    available, budget,
19849                    "contention must not shorten verification"
19850                );
19851                (Vec::new(), false, Vec::new())
19852            },
19853        )
19854        .await
19855        .unwrap();
19856        release.await.unwrap();
19857        assert!(outcomes.is_empty(), "{outcomes:?}");
19858        assert!(
19859            st.events
19860                .iter()
19861                .any(|event| event.message.contains("waiting for"))
19862        );
19863        assert!(st.resource_wait.is_none());
19864        // The verification body no longer owns the lease after returning.
19865        hold(&cache).release();
19866    }
19867
19868    #[tokio::test]
19869    async fn verification_cache_wait_expires_without_starting_commands() {
19870        crate::run::pin_test_home();
19871        let tmp = tempfile::tempdir().unwrap();
19872        let cache = tmp.path().join("target");
19873        let _held = hold(&cache);
19874        let mut st = state(0);
19875        let wait = repository_wait(&st, None);
19876        let (outcomes, _) = with_cache_lease(
19877            &mut st,
19878            Some(&cache),
19879            "gate",
19880            "gate",
19881            tmp.path(),
19882            "abc1234",
19883            Duration::from_secs(60),
19884            "test gate",
19885            &wait,
19886            |_, _| async { panic!("commands must not start without the cache lease") },
19887        )
19888        .await
19889        .unwrap();
19890        assert_eq!(outcomes.len(), 1);
19891        assert!(outcomes[0].resource_blocked);
19892        assert!(st.resource_wait.is_none());
19893        assert!(matches!(
19894            st.resource_block.unwrap().cause,
19895            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19896        ));
19897    }
19898
19899    #[tokio::test]
19900    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19901        crate::run::pin_test_home();
19902        let tmp = tempfile::tempdir().unwrap();
19903        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19904        let cache = tmp.path().join("cache");
19905        let mut state = state(21600);
19906        let wait = repository_wait(&state, None);
19907        let (outcomes, _) = with_cache_lease(
19908            &mut state,
19909            Some(&cache),
19910            "gate",
19911            "gate",
19912            tmp.path(),
19913            "head",
19914            Duration::from_secs(60),
19915            "gate",
19916            &wait,
19917            |_, _| async { panic!("freshness failure must not start verification") },
19918        )
19919        .await
19920        .unwrap();
19921        assert_eq!(outcomes.len(), 1);
19922        assert!(outcomes[0].resource_blocked);
19923        let block = state.resource_block.unwrap();
19924        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19925        assert!(!block.description().contains("timed out"));
19926    }
19927
19928    #[tokio::test]
19929    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19930        crate::run::pin_test_home();
19931        let tmp = tempfile::tempdir().unwrap();
19932        let cache = tmp.path().join("cache");
19933        let _held = hold(&cache);
19934        let mut state = state(21600);
19935        let pause = Pause::new();
19936        let interrupt = Pause::new();
19937        let cause = ParkCause::Operator;
19938        pause.park_for(cause.clone());
19939        let prompts = Prompts::default();
19940        let ctx = WaveCtx {
19941            run: "test",
19942            node: "fix",
19943            prompts: &prompts,
19944            cache: Some(&cache),
19945            round: None,
19946            carry_seats: false,
19947            pauses: Some((&pause, &interrupt)),
19948        };
19949        let result = wave(
19950            vec![job(
19951                sh("writer", "exit 99"),
19952                tmp.path(),
19953                Duration::from_secs(1),
19954            )],
19955            Arc::new(Semaphore::new(1)),
19956            &ctx,
19957            &mut state,
19958            0,
19959        )
19960        .await;
19961        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19962        assert_eq!(result[0].1.turns, 0);
19963        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19964        assert_eq!(
19965            state.resource_block.unwrap().cause,
19966            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19967        );
19968        assert!(state.active.is_empty());
19969    }
19970
19971    #[tokio::test]
19972    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19973        crate::run::pin_test_home();
19974        let tmp = tempfile::tempdir().unwrap();
19975        let cache = tmp.path().join("target-a");
19976        let guard = hold(&cache);
19977        // Held for longer than the agent's whole 1s budget.
19978        tokio::spawn(async move {
19979            tokio::time::sleep(Duration::from_millis(1500)).await;
19980            guard.release();
19981        });
19982        let mut st = state(60);
19983        let results = run(
19984            vec![job(
19985                sh("alpha", "echo ok"),
19986                tmp.path(),
19987                Duration::from_secs(1),
19988            )],
19989            Arc::new(Semaphore::new(1)),
19990            Some(&cache),
19991            &mut st,
19992        )
19993        .await;
19994        assert!(
19995            matches!(results[0].2, AgentOutcome::Ok(_)),
19996            "the wait must not be charged to the agent"
19997        );
19998        assert!(st.resource_block.is_none());
19999        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20000        assert!(st.resource_wait.is_none() && st.active.is_empty());
20001    }
20002
20003    #[tokio::test]
20004    #[cfg(unix)]
20005    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20006        use std::os::fd::AsRawFd as _;
20007        crate::run::pin_test_home();
20008        let tmp = tempfile::tempdir().unwrap();
20009        crate::git::git(tmp.path(), &["init"]).await.unwrap();
20010        let common = tmp.path().join(".git");
20011        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20012        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20013        assert_eq!(
20014            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20015            0
20016        );
20017        let mut st = state(1);
20018        st.config.coordination = crate::config::test_coordination();
20019        let mut reader = job(
20020            sh("reader", "echo must-not-run"),
20021            tmp.path(),
20022            Duration::from_secs(30),
20023        );
20024        reader.allow_write = false;
20025        let results = run(
20026            vec![reader.clone()],
20027            Arc::new(Semaphore::new(1)),
20028            None,
20029            &mut st,
20030        )
20031        .await;
20032        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20033        assert_eq!(results[0].1.turns, 0);
20034        assert!(FailClass::of(&results[0].2).is_none());
20035        assert_eq!(
20036            st.resource_block.as_ref().unwrap().kind,
20037            "repository coordination"
20038        );
20039        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20040        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20041        drop(external);
20042        let editor = crate::repo_lock::SeatLease::for_checkout(
20043            tmp.path(),
20044            true,
20045            crate::config::test_coordination().as_ref(),
20046        )
20047        .await
20048        .unwrap();
20049        let pause = Pause::new();
20050        let interrupt = Pause::new();
20051        let prompts = Prompts::default();
20052        let ctx = WaveCtx {
20053            run: "r",
20054            node: "review",
20055            prompts: &prompts,
20056            cache: None,
20057            round: None,
20058            carry_seats: false,
20059            pauses: Some((&pause, &interrupt)),
20060        };
20061        let (results, ()) = tokio::join!(
20062            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20063            async {
20064                tokio::time::sleep(Duration::from_millis(50)).await;
20065                pause.park();
20066            }
20067        );
20068        assert!(
20069            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20070        );
20071        assert_eq!(results[0].1.turns, 0);
20072        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20073        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20074        drop(editor);
20075    }
20076
20077    #[tokio::test]
20078    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20079        crate::run::pin_test_home();
20080        let tmp = tempfile::tempdir().unwrap();
20081        let cache = tmp.path().join("target-b");
20082        let _held = hold(&cache);
20083        let mut st = state(1);
20084        let results = run(
20085            vec![job(
20086                sh("alpha", "echo ok"),
20087                tmp.path(),
20088                Duration::from_secs(1),
20089            )],
20090            Arc::new(Semaphore::new(1)),
20091            Some(&cache),
20092            &mut st,
20093        )
20094        .await;
20095        let (_, seat, out) = &results[0];
20096        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20097        assert!(wave_blocked(&results));
20098        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20099        assert_eq!(seat.turns, 0, "nothing was asked");
20100        assert!(
20101            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20102            "no agent was spawned"
20103        );
20104        assert!(FailClass::of(out).is_none(), "not a failure class");
20105        let block = st.resource_block.as_ref().expect("recorded for the resume");
20106        assert_eq!(block.kind, "shared build cache");
20107        assert!(matches!(
20108            block.cause,
20109            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20110        ));
20111        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20112        assert!(st.active.is_empty());
20113    }
20114
20115    #[tokio::test]
20116    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20117        crate::run::pin_test_home();
20118        let tmp = tempfile::tempdir().unwrap();
20119        let mut st = state(1);
20120        let results = run(
20121            vec![job(
20122                sh("alpha", "echo ok"),
20123                tmp.path(),
20124                Duration::from_secs(1),
20125            )],
20126            Arc::new(Semaphore::new(0)),
20127            None,
20128            &mut st,
20129        )
20130        .await;
20131        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20132        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20133        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20134    }
20135
20136    #[tokio::test]
20137    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20138        crate::run::pin_test_home();
20139        let tmp = tempfile::tempdir().unwrap();
20140        let mut st = state(60);
20141        let results = run(
20142            vec![job(
20143                sh("alpha", "sleep 30"),
20144                tmp.path(),
20145                Duration::from_secs(1),
20146            )],
20147            Arc::new(Semaphore::new(1)),
20148            None,
20149            &mut st,
20150        )
20151        .await;
20152        let out = &results[0].2;
20153        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20154        let class = FailClass::of(out).expect("a failure");
20155        assert_eq!(class, FailClass::Timeout);
20156        assert!(should_hand_over(None, &class), "existing handover policy");
20157        assert!(st.resource_block.is_none());
20158    }
20159}